From 42dd32ab62fd8ef3bf39bf4383ac3af621e472ee Mon Sep 17 00:00:00 2001 From: Nick Bebout Date: Wed, 1 Jun 2022 10:47:18 -0500 Subject: [PATCH] 2.200 --- ChangeLog | 101 +- FAQ.d/FAQ.Admin_Authentication.txt | 3 +- FAQ.d/FAQ.Authentication_failure.txt | 4 +- FAQ.d/FAQ.Capacity_Planning.txt | 55 + FAQ.d/FAQ.Contacts_Calendars.txt | 6 +- FAQ.d/FAQ.David_Tobit.txt | 6 +- FAQ.d/FAQ.Exchange.txt | 4 +- FAQ.d/FAQ.Folders_Selection.txt | 8 +- FAQ.d/FAQ.General.txt | 13 +- FAQ.d/FAQ.Gmail.txt | 125 +- FAQ.d/FAQ.James.txt | 34 + FAQ.d/FAQ.Kerio.txt | 4 +- FAQ.d/FAQ.Office365.txt | 38 +- FAQ.d/FAQ.OnlineUI.txt | 115 +- FAQ.d/FAQ.POP3.txt | 12 +- FAQ.d/FAQ.SSL_errors.txt | 47 +- FAQ.d/FAQ.Two_Ways_Sync.txt | 29 +- FAQ.d/FAQ.XOAUTH2.txt | 4 +- FAQ.d/htaccess.txt | 3 +- INSTALL.d/Dockerfile | 34 +- INSTALL.d/INSTALL.Centos.txt | 25 +- INSTALL.d/INSTALL.Darwin.txt | 21 +- INSTALL.d/INSTALL.Debian.txt | 6 +- INSTALL.d/INSTALL.Docker_build.txt | 38 +- INSTALL.d/INSTALL.OnlineUI.txt | 291 ++--- INSTALL.d/memo_docker | 22 +- Makefile | 122 +- README | 86 +- README_Windows.txt | 10 +- S/external.shtml | 22 +- S/images/logo_imapsync_s1.png | Bin 0 -> 3957 bytes S/images/logo_imapsync_s2.png | Bin 0 -> 3957 bytes S/imapsync_sold_by_country.txt | 208 ++-- S/news.shtml | 43 +- TODO | 18 +- VERSION | 2 +- X/cgi_memo | 218 ++-- X/imapsync_form.js | 8 +- X/imapsync_form_extra.html | 41 +- X/imapsync_form_extra_free.html | 41 +- X/imapsync_online_status.html | 50 + examples/imapsync_example.bat | 39 +- i3 | 7 - imapsync | 1631 +++++++++++++++++++------- index.shtml | 6 +- tests.sh | 114 +- 46 files changed, 2600 insertions(+), 1114 deletions(-) create mode 100644 FAQ.d/FAQ.Capacity_Planning.txt create mode 100644 FAQ.d/FAQ.James.txt create mode 100644 S/images/logo_imapsync_s1.png create mode 100644 S/images/logo_imapsync_s2.png create mode 100644 X/imapsync_online_status.html delete mode 100755 i3 diff --git a/ChangeLog b/ChangeLog index 019b03a..591d185 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,17 +1,110 @@ RCS file: RCS/imapsync,v Working file: imapsync -head: 2.178 +head: 2.200 branch: locks: strict - gilles: 2.178 + gilles: 2.200 access list: symbolic names: keyword substitution: kv -total revisions: 1078; selected revisions: 1078 +total revisions: 1100; selected revisions: 1100 description: ---------------------------- -revision 2.178 locked by: gilles; +revision 2.200 locked by: gilles; +date: 2022/04/04 16:05:55; author: gilles; state: Exp; lines: +14 -10 +typo. "check the certificate server" => "check the server certificate." +---------------------------- +revision 2.199 +date: 2022/04/01 19:45:33; author: gilles; state: Exp; lines: +15 -14 +Enhancement. Added the local ip address for the imap connexions. It can help configuring firewalls to allow the imap source IP. +---------------------------- +revision 2.198 +date: 2022/04/01 11:13:01; author: gilles; state: Exp; lines: +10 -12 +Added SERVER_NAME SERVER_ADDR SERVER_ADMIN variables and values to the output. +---------------------------- +revision 2.197 +date: 2022/03/30 10:43:22; author: gilles; state: Exp; lines: +29 -28 +Bugfix. --tests --testslive was listing 0 folders. +Added message "Use --noemailreport1 to avoid it" after putting the email final report in INBOX. +---------------------------- +revision 2.196 +date: 2022/03/25 20:35:59; author: gilles; state: Exp; lines: +120 -67 +Bugfix. Dates were not displayed under Windows because POSIX::strftime %e is not portable. +Bugfix. Several other places where dates were wrong on Windows. +---------------------------- +revision 2.195 +date: 2022/03/24 10:00:09; author: gilles; state: Exp; lines: +26 -29 +Added warning "parsing headers of folder ... It can take time for huge folders. Be patient." +---------------------------- +revision 2.194 +date: 2022/03/17 11:04:16; author: gilles; state: Exp; lines: +16 -15 +Bugfix. --var HTTP_COOKIE=proximapsync_runs=31 was not working because of the second = +---------------------------- +revision 2.193 +date: 2022/03/15 11:30:02; author: gilles; state: Exp; lines: +101 -105 +Refactor. Deglobalized $debuglist $debugflags +Refactor. Added sync->{ permanentflags2 } +---------------------------- +revision 2.192 +date: 2022/03/11 18:26:06; author: gilles; state: Exp; lines: +88 -46 +Bugfix. Do not allow --skipcrossduplicates and --usecache. Exit EX_USAGE 64 +---------------------------- +revision 2.191 +date: 2022/03/08 10:55:21; author: gilles; state: Exp; lines: +131 -48 +Added --var to pass values from proximapsync for variables REMOTE_ADDR REMOTE_HOST HTTP_REFERER HTTP_USER_AGENT SERVER_SOFTWARE SERVER_PORT HTTP_COOKIE +---------------------------- +revision 2.190 +date: 2022/03/01 21:48:11; author: gilles; state: Exp; lines: +31 -8 +Bugfix. Do not turn on --delete2duplicates when --syncduplicates is on, unless --delete2duplicates is given. +---------------------------- +revision 2.189 +date: 2022/02/28 18:10:10; author: gilles; state: Exp; lines: +9 -11 +Upped ERRORS_MAX_CGI from 20 to 500 +---------------------------- +revision 2.188 +date: 2022/02/26 22:43:55; author: gilles; state: Exp; lines: +182 -100 +Refactor. Changed setlogfile() to be usable for different logs by proximapsync. +---------------------------- +revision 2.187 +date: 2022/02/23 12:01:25; author: gilles; state: Exp; lines: +26 -7 +Documented why total sizes can differ even when the sync is perfect. +---------------------------- +revision 2.186 +date: 2022/02/22 22:39:50; author: gilles; state: Exp; lines: +27 -21 +Bugfix. Redirect STDERR to STDOUT in all cases, --log or --nolog +---------------------------- +revision 2.185 +date: 2022/02/11 21:36:18; author: gilles; state: Exp; lines: +19 -11 +Added option --emailreport1 and --emailreport2. On by default. Use --noemailreport1 and --noemailreport2 to avoid final emails reports in each INBOX. +---------------------------- +revision 2.184 +date: 2022/02/10 17:37:57; author: gilles; state: Exp; lines: +91 -31 +Added final report to source account also. +---------------------------- +revision 2.183 +date: 2022/02/06 13:14:33; author: gilles; state: Exp; lines: +29 -29 +Added folder name in the debug output of header identificators +---------------------------- +revision 2.182 +date: 2022/02/01 20:49:51; author: gilles; state: Exp; lines: +10 -10 +Bugfix. GMT is good but the tests had to be changed also... +---------------------------- +revision 2.181 +date: 2022/02/01 20:19:22; author: gilles; state: Exp; lines: +23 -10 +Bugfix. Use GMT in tests_email_report_message_id +Bugfix. Make the email report w3c validated. Crazy man! +---------------------------- +revision 2.180 +date: 2022/02/01 15:14:51; author: gilles; state: Exp; lines: +373 -30 +Enhancement. Append a final email report on account2 at the end of the synchronization. Will be optionnal soon. +---------------------------- +revision 2.179 +date: 2022/01/26 14:32:19; author: gilles; state: Exp; lines: +134 -139 +README reread, grammarly on it. +Added "Posta inviata" so \Sent with --automap +---------------------------- +revision 2.178 date: 2022/01/12 21:28:37; author: gilles; state: Exp; lines: +75 -13 Added tests_compress_ssl() ---------------------------- diff --git a/FAQ.d/FAQ.Admin_Authentication.txt b/FAQ.d/FAQ.Admin_Authentication.txt index 9c95f1e..deed0fb 100644 --- a/FAQ.d/FAQ.Admin_Authentication.txt +++ b/FAQ.d/FAQ.Admin_Authentication.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.Admin_Authentication.txt,v 1.12 2021/02/19 13:44:18 gilles Exp gilles $ +$Id: FAQ.Admin_Authentication.txt,v 1.13 2022/04/06 10:27:52 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -42,6 +42,7 @@ Known imap software servers supporting imap admin authentication: * Zimbra. See the file FAQ.Zimbra.txt * Kerio See the file FAQ.Kerio.txt * Cyrus-imap. See the file FAQ.Cyrus.txt + * James. See the file FAQ.James.txt * UW-imap. See the file FAQ.General.txt * Sun Java Enterprise System/SunOne/iPlanet. See the file FAQ.General.txt diff --git a/FAQ.d/FAQ.Authentication_failure.txt b/FAQ.d/FAQ.Authentication_failure.txt index 7219a36..ec41351 100644 --- a/FAQ.d/FAQ.Authentication_failure.txt +++ b/FAQ.d/FAQ.Authentication_failure.txt @@ -1,9 +1,9 @@ #!/bin/cat -$Id: FAQ.Authentication_failure.txt,v 1.14 2021/11/05 11:35:07 gilles Exp gilles $ +$Id: FAQ.Authentication_failure.txt,v 1.15 2022/03/03 11:43:55 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ -https://imapsync.lamiral.info/FAQ.d/FAQ.Admin_Authentication.txt +https://imapsync.lamiral.info/FAQ.d/FAQ.Authentication_failure.txt ======================================================================= Imapsync authentication issues diff --git a/FAQ.d/FAQ.Capacity_Planning.txt b/FAQ.d/FAQ.Capacity_Planning.txt new file mode 100644 index 0000000..54a8ad2 --- /dev/null +++ b/FAQ.d/FAQ.Capacity_Planning.txt @@ -0,0 +1,55 @@ +$Id: FAQ.Capacity_Planning.txt,v 1.3 2022/03/22 11:12:25 gilles Exp gilles $ + +This documentation is also available online at +https://imapsync.lamiral.info/FAQ.d/ +https://imapsync.lamiral.info/FAQ.d/FAQ.Capacity_Planning.txt + + +====================================================================== + Imapsync tips for Capacity Planning. +====================================================================== + + +I plan to go to a distributed architecture for the online service. I'm +doing some capacity planning. Imapsync takes memory, cpu and bandwidth +in a relatively deterministic values. + +My current question is: Shall I use +* N 2GB hosts +* N/2 4GB hosts +* N/4 8GB hosts + +Let's do some maths + +The CPU can be an issue. On average, an imapsync run takes 5% of the +overall cpu time for a Intel i5-2300 with 4 cores. It implies 20 +imapsync runs is ok on the current online host before the cpus become +the bottleneck. As a rule of thumb, imapsync takes 20% of a cpu core. + +The RAM can be an issue. On average, an imapsync run takes 250 MB. So +4 imapsync processes per GB is the limit before swapping to disk, +which is a known phenomenum telling when memory becomes the bottleneck. +16 GB allows 64 imapsync processes. + +The Bandwidth I/O can be an issue. The "Average bandwidth rate" value +given by imapsync at the end of a transfer, and also the bandwidth +rate given during the sync on the ETA line, is the total size of all +messages copied divided by the time passed. If imapsync is run between +two foreign imap servers then the total size transferred on the +network link is twice this value, one time when getting the message +from host1, rx, and one time sending the message to host2, tx. + +On average, an imapsync runs at 3 Mbps both ways, rx and tx, so 6 Mbps +in total. So a 100 Mbps symetric link allows 33 imapsync processes +before the link becomes the bottleneck. + +The best minutes observed so far are a global 187 Mpbs rate (23 MiB/s) +on a 100 Mbps symetrical link done by 28 imapsync processes in parallel, +6.7 Mbps per process at that minute. But the best minutes observed for a +single imapsync process is at 103 Mbps, the rx/tx link limit. + + +The harddisk I/O can be an issue. I don't measure it yet because +imapsync doesn't perform heavy I/0 where it runs. Well, I don't know, +no measure is no knowledge, just guesses. + diff --git a/FAQ.d/FAQ.Contacts_Calendars.txt b/FAQ.d/FAQ.Contacts_Calendars.txt index 490ec94..84e94c3 100644 --- a/FAQ.d/FAQ.Contacts_Calendars.txt +++ b/FAQ.d/FAQ.Contacts_Calendars.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.Contacts_Calendars.txt,v 1.15 2021/09/06 14:32:19 gilles Exp gilles $ +$Id: FAQ.Contacts_Calendars.txt,v 1.16 2022/04/05 13:48:52 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -70,8 +70,8 @@ d) Etc. Search the web. There's also specific tools and paid services. e) I plan to make tools for that but so far nothing has began more than this discussion: -http://www.linux-france.org/prj/imapsync_list/msg01797.html -http://www.linux-france.org/prj/imapsync_list/msg01811.html +http://linux-france.tk/prj/imapsync_list/msg01797.html +http://linux-france.tk/prj/imapsync_list/msg01811.html ======================================================================= Q. How can I avoid copying Calendar or Contacts folders? diff --git a/FAQ.d/FAQ.David_Tobit.txt b/FAQ.d/FAQ.David_Tobit.txt index f1fbc70..a83027c 100644 --- a/FAQ.d/FAQ.David_Tobit.txt +++ b/FAQ.d/FAQ.David_Tobit.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.David_Tobit.txt,v 1.4 2019/03/28 17:58:53 gilles Exp gilles $ +$Id: FAQ.David_Tobit.txt,v 1.5 2022/04/05 13:48:52 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -47,8 +47,8 @@ First try above V10 solution since improvments have been made to support Tobit. R. Look at the discussion: -http://www.linux-france.org/prj/imapsync_list/msg00583.html -http://www.linux-france.org/prj/imapsync_list/threads.html#00583 +http://linux-france.tk/prj/imapsync_list/msg00583.html +http://linux-france.tk/prj/imapsync_list/threads.html#00583 patch saved in ./patches/imapsync-1.337_tobit_V6.patch ======================================================================= \ No newline at end of file diff --git a/FAQ.d/FAQ.Exchange.txt b/FAQ.d/FAQ.Exchange.txt index 77e74bd..0ad6166 100644 --- a/FAQ.d/FAQ.Exchange.txt +++ b/FAQ.d/FAQ.Exchange.txt @@ -1,5 +1,5 @@ -$Id: FAQ.Exchange.txt,v 1.79 2021/12/17 11:59:48 gilles Exp gilles $ +$Id: FAQ.Exchange.txt,v 1.80 2022/04/05 13:48:52 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -288,7 +288,7 @@ and I guess it changes over time. R2. For Exchange, in case throttle appears anyway, fix them with: https://docs.microsoft.com/en-us/exchange/change-user-throttling-settings-for-specific-users-exchange-2013-help See also: - http://www.linux-france.org/prj/imapsync_list/msg02072.html + http://linux-france.tk/prj/imapsync_list/msg02072.html (Subject: Re: [imapsync] From Exchange to Dovecot. Speed Date: Mon, 03 Nov 2014 00:14:59 +0100) diff --git a/FAQ.d/FAQ.Folders_Selection.txt b/FAQ.d/FAQ.Folders_Selection.txt index 4bcd5e5..c668f1e 100644 --- a/FAQ.d/FAQ.Folders_Selection.txt +++ b/FAQ.d/FAQ.Folders_Selection.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.Folders_Selection.txt,v 1.11 2020/12/31 11:41:46 gilles Exp gilles $ +$Id: FAQ.Folders_Selection.txt,v 1.12 2022/03/22 11:12:32 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -72,9 +72,9 @@ R. From https://imapsync.lamiral.info/README --nomixfolders : Do not merge folders when host1 is case-sensitive while host2 is not (like Exchange). Only the first - similar folder is synced (example: with folders - "Sent", "SENT" and "sent" - on host1 only "Sent" will be synced to host2). + similar folder is synced. Example: with folders + "Sent", "SENT" and "sent" on host1, only "Sent" + will be synced to host2. --skipemptyfolders : Empty host1 folders are not created on host2. diff --git a/FAQ.d/FAQ.General.txt b/FAQ.d/FAQ.General.txt index 0db9e09..df5b1b5 100644 --- a/FAQ.d/FAQ.General.txt +++ b/FAQ.d/FAQ.General.txt @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: FAQ.General.txt,v 1.244 2021/02/19 13:38:39 gilles Exp gilles $ +# $Id: FAQ.General.txt,v 1.245 2022/04/05 13:55:47 gilles Exp gilles $ ======================================================================= General FAQ for imapsync @@ -372,13 +372,7 @@ Use --useuid then imapsync will avoid dealing with headers. ======================================================================= Q. How can I try imapsync with latest Mail::IMAPClient 3.xx perl module? -Three solutions at least. - -R1 - Look at the script named "i3" in the tarball, it can be used to - run imapsync with the included Mail-IMAPClient-3.39/ wherever you - unpacked the imapsync tarball - -R2 Run: +R1 Run: cpanm Mail::IMAPClient # this uses cpanminus @@ -391,7 +385,7 @@ R2 Run: perl -MCPAN -e "install Mail::IMAPClient" -R3 If you want to install the Perl module locally in a directory +R2 If you want to install the Perl module locally in a directory - Download latest Mail::IMAPClient 3.xx at http://search.cpan.org/dist/Mail-IMAPClient/ @@ -409,7 +403,6 @@ R3 If you want to install the Perl module locally in a directory or if imapsync is in directory /path/ perl -I./Mail-IMAPClient-3.39/lib /path/imapsync ... - ======================================================================= Q. How can I use imapsync with Mail::IMAPClient 2.2.9 perl module? diff --git a/FAQ.d/FAQ.Gmail.txt b/FAQ.d/FAQ.Gmail.txt index 260704d..387eaa0 100644 --- a/FAQ.d/FAQ.Gmail.txt +++ b/FAQ.d/FAQ.Gmail.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.Gmail.txt,v 1.81 2021/10/06 20:21:26 gilles Exp gilles $ +$Id: FAQ.Gmail.txt,v 1.87 2022/02/10 12:42:52 gilles Exp gilles $ This document is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -41,11 +41,16 @@ Q. Can not open imap connection on [imap.gmail.com] Q. I want to sync the Sent folder only, what is it on Gmail? +Q. From Gmail to Gmail, I want to keep Categories. What can I do? + Q. The context is a Gmail account at the source, but not Gmail at the destination. I want cross-duplicates, different Gmail folders/labels generating duplicate messages across folders on the destination, except for the "All Mail" folder. +Q. I can't sync more than 1000 messages in each Gmail folder. + What can I do? + Q. What are the Gmail imap names of the folders "Sent" "Trash" "Drafts" "Junk" "Flagged" and "All"? @@ -236,7 +241,7 @@ imapsync \ --password2 gmailsecret2 \ --gmail1 --gmail2 -Or, replacing what does --gmail1 --gmail2 (values are from imapsync 2.155): +Or, replacing what does --gmail1 --gmail2 (values are from imapsync 2.178): imapsync \ --host1 imap.gmail.com \ @@ -320,7 +325,7 @@ imapsync --host1 mail.oldhost.com \ --password2 gmailsecret \ --gmail2 -Or, replacing what does --gmail2 (values are from imapsync 1.970): +Or, replacing what does --gmail2 (values are from imapsync 2.178): (For Linux only because of the quotes) imapsync --host1 mail.oldhost.com \ @@ -429,19 +434,19 @@ R. Use this example: imapsync \ --user1 foo@gmail.com \ --password1 gmailsecret \ - --host2 localhost \ + --host2 imap.example.org \ --user2 tata \ --password2 tatasecret \ --gmail1 -Or, replacing what does --gmail1 (values are from imapsync 1.970): +Or, replacing what does --gmail1 (values are from imapsync 2.178): imapsync \ --host1 imap.gmail.com \ --ssl1 \ --user1 foo@gmail.com \ --password1 gmailsecret \ - --host2 localhost \ + --host2 imap.example.org \ --user2 tata \ --password2 tatasecret \ --maxbytespersecond 40_000 \ @@ -627,6 +632,73 @@ R2. In fact, by configuration from the user, it can be another folder name presented in IMAP. Use --automap --dry to see this mapping. See below +======================================================================= +Q. From Gmail to Gmail, I want to keep Categories. What can I do? + +R. First, read a document to learn how to play with Gmail Categories: +https://support.google.com/mail/answer/3094499 + +Categories are not presented in IMAP in Gmail, so synchronizing them +with imapsync requires some work. + +Let's say I want to synchronize the category named "Updates". + +1. Create a label "Updates_L", or name it as you want. You can't + create a label named "Updates" like the category name because + Gmail will refuse to create it, saying it's a reserved name. + +2. Select all messages in the category "Updates". + To do so, you can use the string "category:updates" in the + search input text located at the top of the Gmail page. + Notice the case is irrelevant to the search, + "category:updates" equals "Category:Updates". + + Gmail then announces "All 100 messages on this page are selected." + It's not enough, I want all the "Updates" category messages. + So, on the right of this statement, click on the text + "Select all messages that match this search". + Now Gmail announces "All messages in this search are selected." + +3. Apply label "Updates_L" created at step 1 to all the selected + messages. Gmail will present a warning and a + "Confirm bulk action" button. Press "Ok". + +4. Repeat steps 1 to 3 for each category you want to synchronize. + +5. Transfer your mailbox to the new account with imapsync using the + options --gmail1 --gmail2, the labels will be synchronized. + +6. For each of the labels you made for categories, do the steps 7 to 9. + As an example, I take the label "Updates_L" previously used. + +7. Select all messages with the label "Updates_L". + You can use the string "label:Updates_L" in the search input + text located at the top of the Gmail page. + Gmail then announces "All 100 messages on this page are selected." + It's not enough, I want all the "updates_l" labeled messages. + So, on the right of this statement, click on the text + "Select all messages that match this search". + Now Gmail announces "All messages in this search are selected." + (does it sound similar to a previous §?) + +8. Apply the category "Updates" to all the selected + messages. Gmail will present a warning and a + "Confirm bulk action" button. Press "Ok". + If you search where are the categories: the categories are + located with the labels, as if they were labels. + +9. Delete the temporary label. That part is optional. + +Maybe one day Google will make categories optionally available +in IMAP, like any other label and this trick will be obsolete. +Labels can be "Show in IMAP" at +https://mail.google.com/mail/u/0/#settings/labels +Categories are listed there but not possibly "Show in IMAP", not yet :-) + +Thanks to Michael Leggett https://twitter.com/leggett for this +trick. See the story at +https://twitter.com/leggett/status/1484632316697739267?s=20 + ======================================================================= Q. The context is a Gmail account at the source, but not Gmail at the destination. I want cross-duplicates, different Gmail folders/labels @@ -658,6 +730,16 @@ See https://github.com/imapsync/imapsync/pull/266 (I refused it because well maintaining it will be very hard for me and I have other priorities). +======================================================================= +Q. I can't sync more than 1000 messages in each Gmail folder. + What can I do? + +R. It's a Gmail configuration feature that someone set for your +Gmail account. Go to +https://mail.google.com/mail/u/0/?tab=wm#settings/fwdandpop +and select +"Do not limit the number of messages in an IMAP folder (default)" + ======================================================================= Q. What are the Gmail imap names of the folders "Sent" "Trash" "Drafts" "Junk" "Flagged" and "All"? @@ -715,28 +797,33 @@ which is the opposite goal of --usecache implied by --useuid. ======================================================================= -Q. Gmail does not really delete messages in folder [Gmail]/All Mail - What happens? What can I do? +Q. Gmail does not really delete messages in folder [Gmail]/All Mail + What happens? What can I do? R. It's true and explained in Gmail documentation at -https://support.google.com/mail/answer/7401?hl=en +https://support.google.com/mail/answer/7401?hl=en#zippy=%2Cempty-your-trash To really remove messages in folder "[Gmail]/All Mail", they have to be moved to the "Trash" folder and be deleted from "Trash". -Quoting the link above (25 december 2018, happy christmas by the way!) +Quoting the link above (4 february 2022) ... Empty your Trash -If you don't want a message to stay in your Trash for 30 days, -you can permanently delete it. -* On your computer, go to Gmail. -* On the left side of the page, scroll down, - then click More and then Trash. -* Check the box next to messages you want to permanently delete, - then click Delete forever. -* Note: The Delete forever option is only available in the Trash. - You won’t be able to delete items forever from the Search result view. +If you don't want a message to stay in your trash for 30 days, +you can permanently delete it. + +* On your computer, go to Gmail. + +* On the left side of the page, click More. + +* Scroll to Trash. + +* Check the box next to messages you want to permanently delete. + Then click Delete forever. + +* Tip: The "Delete forever" option is only available in the trash. + You won’t be able to delete items permanently from the Search result view. To delete all messages in your trash, click Empty Trash now. diff --git a/FAQ.d/FAQ.James.txt b/FAQ.d/FAQ.James.txt new file mode 100644 index 0000000..f0c8a48 --- /dev/null +++ b/FAQ.d/FAQ.James.txt @@ -0,0 +1,34 @@ +#!/bin/cat +$Id: FAQ.James.txt,v 1.2 2022/04/06 10:28:16 gilles Exp gilles $ + +This documentation is also available online at +https://imapsync.lamiral.info/FAQ.d/ +https://imapsync.lamiral.info/FAQ.d/FAQ.James.txt + + +======================================================================= + Imapsync tips for James. Specific issues and solutions. +======================================================================= + +James: JAVA Apache Mail Enterprise Server + +Questions answered in this FAQ are: + +Q. How to migrate to Dovecot with an admin/MasterUser account? + + +======================================================================= +Q. How to migrate to James with an admin/MasterUser account? + +R. James supports impersonation and is setup via the administratoId key +field stored in /conf/usersrepository.conf + +There are multiple user repository definitions, each using +a different key:value pair to define administratorId + +https://james.apache.org/server/config-users.html + +(Thanks to From Rob Davis for this Q/R) + +======================================================================= +======================================================================= diff --git a/FAQ.d/FAQ.Kerio.txt b/FAQ.d/FAQ.Kerio.txt index 170bea3..f302b2c 100644 --- a/FAQ.d/FAQ.Kerio.txt +++ b/FAQ.d/FAQ.Kerio.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.Kerio.txt,v 1.2 2019/11/15 11:33:14 gilles Exp gilles $ +$Id: FAQ.Kerio.txt,v 1.3 2022/04/05 13:48:52 gilles Exp gilles $ This document is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -17,7 +17,7 @@ Q. How to migrate from Kerio Connect to XXX R. No special options required. See also: -http://www.linux-france.org/prj/imapsync_list/msg01756.html +http://linux-france.tk/prj/imapsync_list/msg01756.html http://web.archive.org/web/20140224041102/https://www.safetynet-it.com/it-support/mac-kerio-server-to-microsoft-exchange-2010-migration-1/ http://web.archive.org/web/20121125074756/http://www.safetynet-it.com/it-support/mac-kerio-server-to-microsoft-exchange-2010-migration-2/ diff --git a/FAQ.d/FAQ.Office365.txt b/FAQ.d/FAQ.Office365.txt index 8a0cc39..fcd6d37 100644 --- a/FAQ.d/FAQ.Office365.txt +++ b/FAQ.d/FAQ.Office365.txt @@ -1,5 +1,5 @@ -$Id: FAQ.Office365.txt,v 1.33 2022/01/05 11:17:21 gilles Exp gilles $ +$Id: FAQ.Office365.txt,v 1.37 2022/04/06 09:58:39 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -112,7 +112,27 @@ d) Delete this app password when the job with imapsync is finished. R3. Also, check a license is assigned to that account in Office365. -R4. Microsoft introduced something called "security defaults" which is enabled +R4. From Dave Pusey +https://github.com/imapsync/imapsync/issues/317#issuecomment-1027776418 + +I quote Dave nearly verbatim: + +"I had created an app password, and security defaults are already off. + +I have now figured out the issue. + +Despite the MS365 and EXO admin centers showing that IMAP and Basic +Auth were all enabled, it turns out that in Oct 2021, Microsoft began +disabling basic auth for all tenants that had have never used it by +that point. There was an item in my Message Center from that date +saying my tenant was being done. + +You can re-enable it for specific protocols (IMAP in this case) using +the diagnostic process detailed at +https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-and-exchange-online-september-2021-update/bc-p/2782230 +(see the section "Limited Opt Out") + +R5. Microsoft introduced something called "security defaults" which is enabled by default for new tenants. One of the rules blocks IMAP access as of imapsync. The funny thing is that you can't disable a single rule of this @@ -122,15 +142,15 @@ Switching the whole thing off allows the IMAP login. Also, disable double-step authentication on the Azure/Active Directory portal. See here: https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/concept-fundamentals-security-defaults -Thanks to Stephan Buhre for this R4 answer. +Thanks to Stephan Buhre for this R5 answer. -R5. Are there special characters in the password? +R6. Are there special characters in the password? https://imapsync.lamiral.info/FAQ.d/FAQ.Authentication_failure.txt https://imapsync.lamiral.info/FAQ.d/FAQ.Passwords_on_Windows.txt https://imapsync.lamiral.info/FAQ.d/FAQ.Passwords_on_Unix.txt https://imapsync.lamiral.info/FAQ.d/FAQ.Passwords_on_Mac.txt -R6. Triple check the hostname then. Try all of these: +R7. Triple check the hostname then. Try all of these: * outlook.office365.com * imap-mail.outlook.com * imap.outlook.com @@ -570,7 +590,7 @@ Note from Yago Torres Fernandez: (a working command using admin/authuser on host2 Office 365) imapsync ... --authuser2 user_admin@domain.com --user2 user_to_be_migrated@domain.com ^ - --password2 XXXX --ssl2 ^ + --password2 XXXX --ssl2 but previously in Office365, you must do something like the following, using Powershell: @@ -581,6 +601,10 @@ Note from Betsy Lawlor: You can use global modern authentication with two factor on Exchange Online (M365) but you must have "AllowBasicAuthImap" on the admin account you are using to migrate the mail. +Note from Guido (5 April 2022):The way I fixed it was by turning off security defaults +https://docs.microsoft.com/nl-nl/azure/active-directory/fundamentals/concept-fundamentals-security-defaults. +You still need to check IMAP access on an account-basis though. + Remark: PLAIN authentication is the only way to go with --authuser1 for now. So don't use --authmech1 SOMETHING with --authuser1 admin_user, it will not work. @@ -606,7 +630,7 @@ but not for the admin. Example: imapsync ... --authuser2 user_admin@domain.com --user2 user_to_be_migrated See also: -http://www.linux-france.org/prj/imapsync_list/msg02203.html +http://linux-france.tk/prj/imapsync_list/msg02203.html Subject: RE: [imapsync] Office 365 - 'Master User'? Date: Mon, 1 Jun 2015 17:53:54 +0000 diff --git a/FAQ.d/FAQ.OnlineUI.txt b/FAQ.d/FAQ.OnlineUI.txt index 27f9f0e..d643d4d 100644 --- a/FAQ.d/FAQ.OnlineUI.txt +++ b/FAQ.d/FAQ.OnlineUI.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.OnlineUI.txt,v 1.31 2021/12/24 21:02:52 gilles Exp gilles $ +$Id: FAQ.OnlineUI.txt,v 1.32 2022/03/23 11:00:27 gilles Exp gilles $ This document is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -12,23 +12,38 @@ https://imapsync.lamiral.info/FAQ.d/FAQ.OnlineUI.txt Questions answered in this FAQ are: +Q. Can I launch several imap synchronizations on the visual + user interface /X? + Q. How secure is the online visual user interface /X? Q. Does the online service store any sensitive information -like my passwords? + like my passwords? + +Q. I want to switch from the visual interface /X to the + imapsync command line or to the script examples + https://imapsync.lamiral.info/#DOC_BASIC + What should I know? Q. Shall I have issues when the browser times out? -What happens if the browser connection is closed for whatever reason? + What happens if the browser connection is closed for whatever reason? Q. Shall I have issues when the webserver times out? What happens -if the web server closes the connection for whatever reason? + if the web server closes the connection for whatever reason? Q. The sync stalls at the beginning, just after a line like: -"Host1: xxx says it has CAPABILITY for AUTHENTICATE LOGIN" -What is the problem? + "Host1: xxx says it has CAPABILITY for AUTHENTICATE LOGIN" + What is the problem? Now the questions again with their answers. +===================================================================== +Q. Can I launch several imap synchronizations on the visual + user interface /X? + +R. Yes. Open several tabs/windows on your browser and fill each one + with different credentials. + ===================================================================== Q. How secure is the online visual user interface /X? @@ -114,7 +129,7 @@ started its job. ===================================================================== Q. Does the online service store any sensitive information -like my passwords? + like my passwords? No. @@ -125,14 +140,71 @@ you click on the "Sync or resync" green button. So, if you don't want your browser to keep your password, enter a wrong one or empty the password field and click on the "Sync or resync" green button, the wrong password will be saved in your browser and no sync will be done, -since a good login is mandatory to do anything on your mailboxes. +since a correct login is mandatory to do anything on your mailboxes. + +===================================================================== +Q. I want to switch from the visual interface /X to the + imapsync command line or to the script examples + https://imapsync.lamiral.info/#DOC_BASIC + What should I know? + +R. Let's do some ascii art. + The visual interface looks roughly like this, for the textfields input part: + ++------------------------------------------+------------------------------------------+ +| IMAP source Mailbox | IMAP destination Mailbox | +| | | +| Login (usually an email address) | Login (usually an email address) | +| | | +| test1 | test2 | +| | | +| Password | Password | +| | | +| secret1 | secret2 | +| | | +| IMAP Server hostname (or its IP address) | IMAP Server hostname (or its IP address) | +| | | +| test1.lamiral.info | test2.lamiral.info | +| | | ++------------------------------------------+------------------------------------------+ + +Notice the 6 examples values I put in the previous "picture": + +* test1 +* secret1 +* test1.lamiral.info + +* test2 +* secret2 +* test2.lamiral.info + + +The corresponding imapsync command line on Windows is: + +imapsync.exe --host1 "test1.lamiral.info" --user1 "test1" --password1 "secret1" ^ + --host2 "test2.lamiral.info" --user2 "test2" --password2 "secret2" + +The order of the parameters is whatever you want as long as you respect each pair +like: --optionname "value" + +There is no need to type this in a command prompt window, just use the example script +https://imapsync.lamiral.info/examples/imapsync_example.bat + + +The corresponding imapsync command line on Linux is: + +imapsync --host1 "test1.lamiral.info" --user1 "test1" --password1 "secret1" \ + --host2 "test2.lamiral.info" --user2 "test2" --password2 "secret2" + +There is no need to type this in a command prompt window, just use the example script +https://imapsync.lamiral.info/examples/imapsync_example.sh ===================================================================== Q. Shall I have issues with the browser timing out? -What happens if the browser connection is closed for whatever reason? + What happens if the browser connection is closed for whatever reason? R. A browser connection closed closes also the imapsync process, -ie, the sync is ended right away. + ie, the sync is ended right away. Further comments on this behavior. @@ -145,7 +217,7 @@ If the Browser/WebServer connection timeouts or ends, the imapsync sync is also ended immediately by the remote Apache HTTPS server. Technically, Apache sends a TERM signal to the imapsync process, then wait some seconds before -sending a KILL signal if it is still alive. +sending a KILL signal if the imapsync process is still alive. You can relaunch the sync with the "Sync!" button, at any time. If the "Sync!" button is gray/inactive then just reload @@ -166,26 +238,23 @@ the logfile running the sync like a "tail -f" command (isn't that magic?). ===================================================================== Q. Shall I have issues when the webserver times out? What happens -if the web server closes the connection for whatever reason? + if the web server closes the connection for whatever reason? R. If the webserver closes the connection then usually it also -kills the imapsync process and the imap connections as well. + kills the imapsync process and the imap connections as well. -To fix this issue, see the document INSTALL.OnlineUI.txt -and search for "Timeout" in it. - -Tip: If the connections close for huge folders and always after -300 seconds or 60 seconds then it looks like a web server timeout. -Apache 2.2 timeout is 300 seconds. -Apache 2.4 timeout is 60 seconds. +The current webserver timeout at +https://imapsync.lamiral.info/X/ +is 3600 secondes, one hour. ===================================================================== Q. The sync stalls at the beginning, just after a line like: -"Host1: xxx says it has CAPABILITY for AUTHENTICATE LOGIN" -What is the problem? + "Host1: xxx says it has CAPABILITY for AUTHENTICATE LOGIN" + What is the problem? R. I've seen this issue on /X with the imap server -CommuniGate Pro IMAP Server 6.0.11 + CommuniGate Pro IMAP Server 6.0.11 + The issue looks related to special characters in the password. Solution for now: change the password, keep only standard (ASCII) alphanumeric characters ABC-YZ abc-yz 012-89. diff --git a/FAQ.d/FAQ.POP3.txt b/FAQ.d/FAQ.POP3.txt index e557392..5f0f913 100644 --- a/FAQ.d/FAQ.POP3.txt +++ b/FAQ.d/FAQ.POP3.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.POP3.txt,v 1.5 2019/07/27 20:18:37 gilles Exp gilles $ +$Id: FAQ.POP3.txt,v 1.6 2022/04/05 13:48:52 gilles Exp gilles $ This document is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -24,8 +24,8 @@ Q. Can I use imapsync to migrate emails from pop3 server to imap server? R1. No, but you can migrate emails from a pop3 server to an imap server with the command line tool pop2imap: -http://www.linux-france.org/prj/pop2imap/ -http://www.linux-france.org/prj/pop2imap/README +http://linux-france.tk/prj/pop2imap/ +http://linux-france.tk/prj/pop2imap/README R2. Yes, sometimes, because many pop3 servers runs in parallel with an imap server on exactly the same mailboxes. They serve @@ -81,10 +81,10 @@ R4. Here are some points, an analyse that made me purpose the not to bad but not very good R2 answer. There was a discussion about this issue on the imapsync mailing-list: -http://www.linux-france.org/prj/imapsync_list/msg02622.html -http://www.linux-france.org/prj/imapsync_list/msg02623.html +http://linux-france.tk/prj/imapsync_list/msg02622.html +http://linux-france.tk/prj/imapsync_list/msg02623.html My reply was not very good -http://www.linux-france.org/prj/imapsync_list/msg02624.html +http://linux-france.tk/prj/imapsync_list/msg02624.html It's because I wrote pop2imap and it uses a stateless mechanism to avoid duplicates, I wasn't aware of UIDL when I wrote pop2imap, and later when I entered the discussion. diff --git a/FAQ.d/FAQ.SSL_errors.txt b/FAQ.d/FAQ.SSL_errors.txt index 265ea25..58730e9 100644 --- a/FAQ.d/FAQ.SSL_errors.txt +++ b/FAQ.d/FAQ.SSL_errors.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.SSL_errors.txt,v 1.17 2021/08/01 12:49:00 gilles Exp gilles $ +$Id: FAQ.SSL_errors.txt,v 1.18 2022/01/14 21:20:37 gilles Exp gilles $ This document is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -12,31 +12,52 @@ https://imapsync.lamiral.info/FAQ.d/FAQ.SSL_errors.txt Questions answered in this FAQ are: +Q. What is the error + DEBUG: .../IO/Socket/SSL.pm:1177: global error: Undefined SSL object + Q. What are the errors - DEBUG: .../IO/Socket/SSL.pm:1165: local error: SSL write error - or - DEBUG: .../IO/Socket/SSL.pm:1088: local error: SSL read error + DEBUG: .../IO/Socket/SSL.pm:1165: local error: SSL write error + or + DEBUG: .../IO/Socket/SSL.pm:1088: local error: SSL read error Q. What can I do to avoid those "SSL read/write errors"? -Q. SSL connect attempt failed SSL +Q. What are the errors + SSL connect attempt failed SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure or SSL connect attempt failed SSL SSL routines:ssl_choose_client_version:unsupported protocol -Q. fatal SSL error: SSL connect attempt failed with unknown error +Q. What is the error + fatal SSL error: SSL connect attempt failed with unknown error SSL wants a read first Q. How to see the certificate and identify problems in it? Now the questions again with their answers. +====================================================================== +Q. What is the error + DEBUG: .../IO/Socket/SSL.pm:1177: global error: Undefined SSL object + +R. It's a fake error from the Perl Module IO::Socket::SSL + Imapsync works well despite this fake warning but it's disturbing + when you encounter errors due to something else, you believe it's + the issue but no, it's something else to deal with. + + This fake error is fixed in IO::Socket::SSL release 2.073 + https://metacpan.org/dist/IO-Socket-SSL/changes + "fix #110 - prevent internal error warning in some cases" + https://github.com/noxxi/p5-io-socket-ssl/issues/110 + + imapsync.exe release 2.178 uses this fixed 2.073 IO::Socket::SSL + ====================================================================== Q. What are the errors - DEBUG: .../IO/Socket/SSL.pm:1165: local error: SSL write error - or - DEBUG: .../IO/Socket/SSL.pm:1088: local error: SSL read error + DEBUG: .../IO/Socket/SSL.pm:1165: local error: SSL write error + or + DEBUG: .../IO/Socket/SSL.pm:1088: local error: SSL read error R1. As they claim, those errors are SSL errors. SSL is not directly @@ -74,7 +95,8 @@ R5. Set up a ssltunnel proxy to the host. a ssltunnel proxy. ====================================================================== -Q. SSL connect attempt failed SSL +Q. What are the errors + SSL connect attempt failed SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure or SSL connect attempt failed SSL @@ -149,7 +171,8 @@ In this case setting the version to 'SSLv23:!SSLv2:!SSLv3:!TLSv1_1:!TLSv1_2' might help. ====================================================================== -Q. fatal SSL error: SSL connect attempt failed with unknown error +Q. What is the error + fatal SSL error: SSL connect attempt failed with unknown error SSL wants a read first R. If you're using --ssl1 or --ssl2, try instead --tls1 or --tls2 @@ -201,7 +224,6 @@ ggEBAMUTJVdrTl86nDI2yO6Vz5l1qxMMPqJylQcgi9vDHpwsnUq5HGPv+qZNhM69 ... - After an complete server update ("apt update && apt upgrade && /etc/init.d/dovecot restart"): echo | openssl s_client -crlf -connect test1.lamiral.info:993 @@ -228,6 +250,7 @@ EnRlc3QxLmxhbWlyYWwuaW5mbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC ggEBANuPNbYLPMZ4vPa9NBoHAUdIXqpi0eqdXMXd2sT+qRmqxS5ihr999BHOROcr ... +Champagne! ====================================================================== ====================================================================== diff --git a/FAQ.d/FAQ.Two_Ways_Sync.txt b/FAQ.d/FAQ.Two_Ways_Sync.txt index 32c1eee..2650088 100644 --- a/FAQ.d/FAQ.Two_Ways_Sync.txt +++ b/FAQ.d/FAQ.Two_Ways_Sync.txt @@ -1,4 +1,4 @@ -$Id: FAQ.Two_Ways_Sync.txt,v 1.5 2021/06/10 11:21:09 gilles Exp gilles $ +$Id: FAQ.Two_Ways_Sync.txt,v 1.6 2022/02/22 15:16:16 gilles Exp gilles $ This documentation is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -6,18 +6,9 @@ https://imapsync.lamiral.info/FAQ.d/FAQ.Two_Ways_Sync.txt ======================================================================= - Two ways sync with Imapsync +================ Two ways sync with Imapsync ========================= ======================================================================= - -Questions answered in this FAQ are: - -Q. Can Imapsync do a good "two ways" sync? - Short answer: no, not a good one. Why? - - -Now the questions again with their answers. - ======================================================================= Q. Can Imapsync do a good "two ways" sync? Short answer: no, not a good one. Why? @@ -52,16 +43,14 @@ run a sync with the --delete2 option from A to B. If you know that the missing messages on A are missing messages from B that has to be copied to A then run a sync from B to A. -If you know it's a mixed scenario, some deletions/moves on A, -and some deletions/moves on B, but not the same, then you are in -trouble and so you end up with a not very good "two ways" sync. -I suggest to avoid imapsync to do deletions in that case, which is -the default imapsync behavior. +If you know it's a mixed scenario, that there are some deletions/moves +on A, and there are some deletions/moves on B, but not the same, then +you are in trouble and it ends up with a not very good "two ways" +sync. In that case, two runs of imapsync, one from A to B, one from B +to A, put the two accounts in a synchronized state. But deletions on +either side get cancelled and folder renamings on either side bring +messages duplicates. -With a two ways sync, the account user is very surprised and -disapointed when his actions (deletions, renamings, or movings) come -back: the deletions are cancelled, the renamings and movings end up -with duplicates. ======================================================================= ======================================================================= diff --git a/FAQ.d/FAQ.XOAUTH2.txt b/FAQ.d/FAQ.XOAUTH2.txt index 1ad186d..ba5ea48 100644 --- a/FAQ.d/FAQ.XOAUTH2.txt +++ b/FAQ.d/FAQ.XOAUTH2.txt @@ -1,5 +1,5 @@ #!/bin/cat -$Id: FAQ.XOAUTH2.txt,v 1.17 2021/01/21 13:25:02 gilles Exp gilles $ +$Id: FAQ.XOAUTH2.txt,v 1.18 2022/04/05 13:48:52 gilles Exp gilles $ This document is also available online at https://imapsync.lamiral.info/FAQ.d/ @@ -154,7 +154,7 @@ also called cpanminus. The code and first explanation comes from Joaquin Lopez at https://github.com/imapsync/imapsync/pull/25 -http://www.linux-france.org/prj/imapsync_list/msg02129.html +http://linux-france.tk/prj/imapsync_list/msg02129.html Also, the binary command "openssl" is needed since it is used to convert the pk12 file. diff --git a/FAQ.d/htaccess.txt b/FAQ.d/htaccess.txt index cc109e3..11c8635 100644 --- a/FAQ.d/htaccess.txt +++ b/FAQ.d/htaccess.txt @@ -1,5 +1,5 @@ -# $Id: htaccess.txt,v 1.32 2021/07/23 12:38:19 gilles Exp gilles $ +# $Id: htaccess.txt,v 1.33 2022/04/06 10:27:52 gilles Exp gilles $ AddDescription "Back to Imapsync main page." .. @@ -43,6 +43,7 @@ AddDescription "Gmail accounts." FAQ.Gmai AddDescription "GoDaddy accounts." FAQ.GoDaddy.txt AddDescription "IceWarp accounts." FAQ.IceWarp.txt AddDescription "ISP tips." FAQ.ISP.txt +AddDescription "James accounts." FAQ.James.txt AddDescription "Kerio accounts." FAQ.Kerio.txt AddDescription "MailEnable accounts." FAQ.MailEnable.txt AddDescription "Massive and bulk migrations." FAQ.Massive.txt diff --git a/INSTALL.d/Dockerfile b/INSTALL.d/Dockerfile index 2805161..dfbeb2e 100644 --- a/INSTALL.d/Dockerfile +++ b/INSTALL.d/Dockerfile @@ -1,24 +1,38 @@ ## Dockerfile for building a docker imapsync image -# $Id: Dockerfile,v 1.38 2021/11/29 10:20:56 gilles Exp gilles $ +# $Id: Dockerfile,v 1.42 2022/04/04 19:16:50 gilles Exp gilles $ # I use the following command to build the image: # # docker build -t gilleslamiral/imapsync . # # where this Dockerfile is in the current directory # +# This Dockerfile build an image with two imapsync +# 1) One comes from https://imapsync.lamiral.info/imapsync +# and goes to /usr/bin/imapsync in the Docker image +# It is used with the command: +# +# docker run gilleslamiral/imapsync imapsync ... +# +# 2) One comes from the local file ./imapsync +# and goes to /imapsync in the Docker image +# It is used with the command: +# +# docker run gilleslamiral/imapsync /imapsync ... + + # I thank you very much # I like thanks # I like stars -# I also need money to keep on doing this stuff +# I like money to keep on doing this stuff -# Number of imapsync images pulled so far (2021_11_29): 608326 +# Number of imapsync images pulled so far (2022_04_04): 673551 # Command used: # curl -s https://hub.docker.com/v2/repositories/gilleslamiral/imapsync/ | jq '.pull_count' -# Debian Buster is Debian 10 +# Debian Bullseye is Debian 11 -FROM debian:buster +FROM debian:bullseye LABEL maintainer="Gilles LAMIRAL " \ description="Imapsync" \ @@ -30,8 +44,6 @@ LABEL maintainer="Gilles LAMIRAL " \ COPY Dockerfile imapsyn[c] prerequisites_imapsyn[c] / - - RUN set -xe && \ apt-get update \ && apt-get install -y \ @@ -69,7 +81,11 @@ RUN set -xe && \ ncat \ openssl \ ca-certificates \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && cpanm IO::Socket::SSL + +# I added the IO::Socket::SSL update to avoid the annoying, confusing and useless warning +# DEBUG: .../IO/Socket/SSL.pm:1177: global error: Undefined SSL object RUN set -xe \ && cd /usr/bin/ \ @@ -78,7 +94,7 @@ RUN set -xe \ https://imapsync.lamiral.info/prerequisites_imapsync \ https://raw.githubusercontent.com/google/gmail-oauth2-tools/master/python/oauth2.py \ && chmod +x imapsync oauth2.py \ - && /usr/bin/imapsync --testslive && /usr/bin/imapsync --tests # just_a_comment_to_force_update 2021_11_29_11_20_56 + && /usr/bin/imapsync --testslive && /usr/bin/imapsync --tests # just_a_comment_to_force_update 2022_04_04_21_16_50 USER nobody:nogroup diff --git a/INSTALL.d/INSTALL.Centos.txt b/INSTALL.d/INSTALL.Centos.txt index 7f047b4..c25bea7 100644 --- a/INSTALL.d/INSTALL.Centos.txt +++ b/INSTALL.d/INSTALL.Centos.txt @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: INSTALL.Centos.txt,v 1.29 2021/12/17 14:52:20 gilles Exp gilles $ +# $Id: INSTALL.Centos.txt,v 1.31 2022/02/19 20:28:13 gilles Exp gilles $ This documentation is also located online at https://imapsync.lamiral.info/INSTALL.d/ @@ -24,6 +24,29 @@ How to install imapsync on AlmaLinux and Centos 8: imapsync --testslive +For Centos 8 End Of Life fix: +https://www.centos.org/centos-linux-eol/ + +If you encounter the error +"Error: Failed to download metadata for repo 'appstream': Cannot prepare internal mirrorlist: No URLs in mirrorlist" + +Do this: + + sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* + dnf update + +To upgrade to CentOS stream: + + dnf install centos-release-stream + dnf swap centos-{linux,stream}-repos + dnf distro-sync + +Taken from +https://stackoverflow.com/questions/70926799/centos-through-vm-no-urls-in-mirrorlist + +and tested successfuly on one of my hosts, an ex-Centos8/CentOS-stream now. + ======================================================================= AlmaLinux and Centos 8 and latest imapsync ======================================================================= diff --git a/INSTALL.d/INSTALL.Darwin.txt b/INSTALL.d/INSTALL.Darwin.txt index 7813c2f..b01c553 100644 --- a/INSTALL.d/INSTALL.Darwin.txt +++ b/INSTALL.d/INSTALL.Darwin.txt @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: INSTALL.Darwin.txt,v 1.36 2021/12/24 21:08:13 gilles Exp gilles $ +# $Id: INSTALL.Darwin.txt,v 1.38 2022/04/05 19:44:30 gilles Exp gilles $ This documentation is also located online at https://imapsync.lamiral.info/INSTALL.d/ @@ -20,21 +20,20 @@ E) Building imapsync binary on Darwin / Mac OS X A) Installing imapsync binary on Darwin / Mac OS X ======================================================================= - There is a standalone imapsync binary for Mac OS X called "imapsync_bin_Darwin" (without the quotes), -available in the compressed tarball called imapsync-2.140.tgz -where 2.140 is the imapsync version number. +available in the compressed tarball called imapsync-2.200.tgz +where 2.200 is the imapsync version number. In case you haven't buy it yet or forgot where the download link is, check https://imapsync.lamiral.info/dist/ -Download the tarball imapsync-2.140.tgz, not the binary imapsync_bin_Darwin +Download the tarball imapsync-2.200.tgz, not the binary imapsync_bin_Darwin because the binary is also in the tarball and there are other useful files in the tarball. The binary present in /dist/ is just there for a fast upgrade when you need one. -I suppose this tarball imapsync-2.140.tgz is downloaded under your $HOME directory, +I suppose this tarball imapsync-2.200.tgz is downloaded under your $HOME directory, let say /Users/gilles/, but you can put it anywhere. In real, your $HOME directory is not /Users/gilles/, it maybe /Users/john/ or /Users/zoey/ where John or Zoey is you login name. @@ -45,19 +44,19 @@ Untar the tarball: cd pwd - tar xzvf imapsync-2.140.tgz + tar xzvf imapsync-2.200.tgz -In case the previous command fails, it means the tarball file called imapsync-2.140.tgz +In case the previous command fails, it means the tarball file called imapsync-2.200.tgz is not in your $HOME directory, you may have downloaded it elsewhere on the file system. A way to find it is the command: find / | grep imapsync -Now that the tarball is extracted, it created a directory called imapsync-2.140/ -Go into the directory imapsync-2.140 with the command: +Now that the tarball is extracted, it created a directory called imapsync-2.200/ +Go into the directory imapsync-2.200 with the command: - cd imapsync-2.140 + cd imapsync-2.200 First let's have a simple run to see if imapsync_bin_Darwin works. diff --git a/INSTALL.d/INSTALL.Debian.txt b/INSTALL.d/INSTALL.Debian.txt index 5e5ac64..87f9c00 100644 --- a/INSTALL.d/INSTALL.Debian.txt +++ b/INSTALL.d/INSTALL.Debian.txt @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: INSTALL.Debian.txt,v 1.35 2021/12/17 20:30:33 gilles Exp gilles $ +# $Id: INSTALL.Debian.txt,v 1.37 2022/03/29 11:03:55 gilles Exp gilles $ This documentation is also located online at https://imapsync.lamiral.info/INSTALL.d/ @@ -8,7 +8,7 @@ https://imapsync.lamiral.info/INSTALL.d/INSTALL.Debian.txt There is four install sections in this document, 1) one for Debian 9 Stretch, Debian 10 Buster, Debian 11 Bullseye -2) one for all the Debian called "After installing the dependencies" +2) one for all the Debian, called "After installing the dependencies". 3) one for Debian 8 Jessie (obsolete) 4) one for Debian 7 Wheezy (even more obsolete) @@ -110,7 +110,7 @@ You don't have to be root to test and use imapsync. Take imapsync either on github: - wget -N https://github.com/imapsync/imapsync/blob/master/imapsync + wget -N https://raw.githubusercontent.com/imapsync/imapsync/master/imapsync or be on the cutting edge with the upstream site: diff --git a/INSTALL.d/INSTALL.Docker_build.txt b/INSTALL.d/INSTALL.Docker_build.txt index 814c83a..aee93e9 100644 --- a/INSTALL.d/INSTALL.Docker_build.txt +++ b/INSTALL.d/INSTALL.Docker_build.txt @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: INSTALL.Docker_build.txt,v 1.15 2021/11/29 11:56:11 gilles Exp gilles $ +# $Id: INSTALL.Docker_build.txt,v 1.16 2022/04/05 13:20:24 gilles Exp gilles $ This documentation is also located online at https://imapsync.lamiral.info/INSTALL.d/ @@ -14,25 +14,39 @@ Nearly verbatim copy from https://imapsync.lamiral.info/INSTALL.d/Dockerfile ## Dockerfile for building a docker imapsync image -# Id: Dockerfile,v 1.38 2021/11/29 10:20:56 gilles Exp gilles $ +# Id: Dockerfile,v 1.42 2022/04/04 19:16:50 gilles Exp gilles $ # I use the following command to build the image: # # docker build -t gilleslamiral/imapsync . # # where this Dockerfile is in the current directory # +# This Dockerfile build an image with two imapsync +# 1) One comes from https://imapsync.lamiral.info/imapsync +# and goes to /usr/bin/imapsync in the Docker image +# It is used with the command: +# +# docker run gilleslamiral/imapsync imapsync ... +# +# 2) One comes from the local file ./imapsync +# and goes to /imapsync in the Docker image +# It is used with the command: +# +# docker run gilleslamiral/imapsync /imapsync ... + + # I thank you very much # I like thanks # I like stars -# I also need money to keep on doing this stuff +# I like money to keep on doing this stuff -# Number of imapsync images pulled so far (2021_11_29): 608326 +# Number of imapsync images pulled so far (2022_04_04): 673551 # Command used: # curl -s https://hub.docker.com/v2/repositories/gilleslamiral/imapsync/ | jq '.pull_count' -# Debian Buster is Debian 10 +# Debian Bullseye is Debian 11 -FROM debian:buster +FROM debian:bullseye LABEL maintainer="Gilles LAMIRAL " \ description="Imapsync" \ @@ -44,8 +58,6 @@ LABEL maintainer="Gilles LAMIRAL " \ COPY Dockerfile imapsyn[c] prerequisites_imapsyn[c] / - - RUN set -xe && \ apt-get update \ && apt-get install -y \ @@ -83,7 +95,11 @@ RUN set -xe && \ ncat \ openssl \ ca-certificates \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && cpanm IO::Socket::SSL + +# I added the IO::Socket::SSL update to avoid the annoying, confusing and useless warning +# DEBUG: .../IO/Socket/SSL.pm:1177: global error: Undefined SSL object RUN set -xe \ && cd /usr/bin/ \ @@ -92,7 +108,7 @@ RUN set -xe \ https://imapsync.lamiral.info/prerequisites_imapsync \ https://raw.githubusercontent.com/google/gmail-oauth2-tools/master/python/oauth2.py \ && chmod +x imapsync oauth2.py \ - && /usr/bin/imapsync --testslive && /usr/bin/imapsync --tests # just_a_comment_to_force_update 2021_11_29_11_20_56 + && /usr/bin/imapsync --testslive && /usr/bin/imapsync --tests # just_a_comment_to_force_update 2022_04_04_21_16_50 USER nobody:nogroup @@ -107,7 +123,5 @@ CMD ["/usr/bin/imapsync"] # # End of imapsync Dockerfile - - ======================================================================= ======================================================================= diff --git a/INSTALL.d/INSTALL.OnlineUI.txt b/INSTALL.d/INSTALL.OnlineUI.txt index 67b406f..6c943f5 100644 --- a/INSTALL.d/INSTALL.OnlineUI.txt +++ b/INSTALL.d/INSTALL.OnlineUI.txt @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: INSTALL.OnlineUI.txt,v 1.50 2021/09/20 13:47:20 gilles Exp gilles $ +# $Id: INSTALL.OnlineUI.txt,v 1.52 2022/03/23 11:02:05 gilles Exp gilles $ This documentation is also located online at https://imapsync.lamiral.info/INSTALL.d/ @@ -26,26 +26,25 @@ Load mean: 0.8 on a CPU 4 cores "Intel(R) i5-2320 3.00GHz K8-class" ======================================================================= Installation -You have to be a little familiar with what a CGI script is -and how to activate a CGI script on the Apache -HTTP server, or any other HTTP server. I have received demands -to run it on the Ngnix HTTP server but I haven't played with it yet. -Linux is also a preferred platform (I run /X service on Linux and FreeBSD). +You have to be a little familiar with what a CGI script is and how to +activate a CGI script on the Apache HTTP server, or any other HTTP +server. I have received demands to run it on the Ngnix HTTP server but +I haven't played with it yet. Linux is also a preferred platform (I +run /X service on Linux and FreeBSD). -I have tested this visual interface on Mac. It works. -For now, it demands some skills few Mac users have. -Drop me a note in case you want to do that. +I have tested this visual interface on Mac. It works. For now, it +demands some skills few Mac users have. Drop me a note in case you +want to do that. -I have tested this visual interface on Windows, -it fails on Windows because of some hardcoded Unix paths. -I'm working on it to be Windows ok but it's not done yet (May 2020). +I have tested this visual interface on Windows, it fails on Windows +because of some hardcoded Unix paths. I'm working on it to be Windows +ok but it's not done yet (May 2020). -Some users have successfully installed a /X visual interface -on Windows using a Linux VM machine. +Some users have successfully installed a /X visual interface on +Windows using a Linux VM machine. -The web visual user interface frontend /X is compounded in four -files: a html5 file, a CSS file, a javascript file, and -a logo image: +The web visual user interface frontend /X is compounded in four files: +a html5 file, a CSS file, a javascript file, and a logo image: * https://imapsync.lamiral.info/X/imapsync_form_extra.html * https://imapsync.lamiral.info/X/imapsync_form.css @@ -53,29 +52,28 @@ a logo image: * https://imapsync.lamiral.info/X/logo_imapsync_Xn.png -You can do a "view source" to see the HTML file as -it is written, and a "save" to get it locally. -The three other files can be saved the same way or -with a command named "wget". I suggest using wget, -see below the ready-to-use command lines. +You can do a "view source" to see the HTML file as it is written, and +a "save" to get it locally. The three other files can be saved the +same way or with a command named "wget". I strongly suggest using +wget, see below the ready-to-use command lines. -Those four files can be put anywhere on a web server, -as long as they stand in the same directory. If you -want to put them in different directories, just change -the content of imapsync_form_extra.html to reflect the change, -ie, change the two lines referencing imapsync_form.css and imapsync_form.js - href="imapsync_form.css" (near the beginning of imapsync_form_extra.html) - src="imapsync_form.js" (near the end of imapsync_form_extra.html) -I let you change the image logo as an exercise, it's safe if you fail. +Those four files can be put anywhere on a web server, as long as they +stand in the same directory. If you want to put them in different +directories, just change the content of imapsync_form_extra.html to +reflect the change, ie, change the two lines referencing +imapsync_form.css and imapsync_form.js href="imapsync_form.css" (near +the beginning of imapsync_form_extra.html) src="imapsync_form.js" +(near the end of imapsync_form_extra.html) I let you change the image +logo as an exercise, it's safe if you fail. -The actual imap syncing work is done by imapsync acting as a CGI, -the visual interface is only there to give imapsync the parameters -needed for the sync. +The actual imap syncing work is done by imapsync acting as a CGI, the +visual interface is only there to give imapsync the parameters needed +for the sync. -Use at least Perl module CGI.pm release 4.08 (2014-10-18) -to avoid the bug "Undefined subroutine CGI::multi_param". -You can use the command named cpanm to upgrade CGI.pm to -its last version, it's the easiest way. +Use at least Perl module CGI.pm release 4.08 (2014-10-18) to avoid the +bug "Undefined subroutine CGI::multi_param". You can use the command +named cpanm to upgrade CGI.pm to its last version, it's the easiest +way. Print the CGI.pm release with: @@ -85,8 +83,12 @@ If it is under release 4.08 (2014-10-18) then upgrade it with cpanm CGI -It is a good thing to remove the old one if it was installed -by a distribution package, I let you this part as an exercise too. +It is a good thing to remove the old one if it was installed by a +distribution package, I let you this part as an exercise too. Ok, here +is a way: + + apt remove libcgi-pm-perl # for the Debian family + dnf remove perl-CGI # for the Centos family To check and fix the Perl modules dependencies, run: @@ -94,50 +96,59 @@ To check and fix the Perl modules dependencies, run: wget -N https://imapsync.lamiral.info/prerequisites_imapsync sh prerequisites_imapsync -To make imapsync work as a CGI script, there are two conditions. -First, imapsync has to work by itself on the web host. +To make imapsync work as a CGI script, there are three conditions. -If imapsync doesn't work by itself, as a command line, -then it won't work as a CGI script. +First, imapsync has to work by itself on the web host. If imapsync +doesn't work by itself, as a command line, then it won't work as a CGI +script. -Second, imapsync has to work by itself on the web host using -the Unix user running the webserver. Detailed examples to -verify that will be provided further in this document. +Second, imapsync has to work by itself on the web host using the Unix +user running the webserver. + +Third, the file imapsync has to be considered as a cgi script. + +Command lines to provide and verify those three conditions will be +provided further in this document, for the Debian family systems and +for the Centos family systems. You are strongly advised to follow this +commands if need and want help from me because I will first ask you to +run them before searching what you did wrong. The imapsync_form_extra.html file in action calls the CGI location /cgi-bin/imapsync which has to be imapsync itself, the file script (not the directory). -The very latest and relatively stable imapsync is at +The very latest and relatively stable imapsync is https://imapsync.lamiral.info/imapsync This file is the program file used verbatim for the service given at https://imapsync.lamiral.info/X/ -Copy the three files imapsync_form.* on a directory that is exported +Copy the three files imapsync_form.* on a directory that is exported by your HTTP server. + Copy the imapsync script on the cgi-bin/ directory allowing CGIs and you'll have your imapsync visual interface and service. The cgi-bin/ directory is usually outside the hierarchy exported to anybody by the HTTP server. -The default Apache 2.4 timeout is 60 seconds, one minute, 300 for older Apache. +The default Apache 2.4 timeout is 60 seconds, one minute, and 300 +secondes for older Apache, 5 minutes. See https://httpd.apache.org/docs/2.4/mod/core.html#timeout -Mine is now "Timeout 3600", 3600 seconds, an hour. I chose this huge -timeout value because imapsync can spend a long time without talking while -getting the headers of huge folders of 100k messages. -If you intend to offer this service for huge mailboxes or for a long -time, I strongly recommand you to set this "Timeout 3600" in -the Apache configuration right now because you will sure end up with this -timeout issue in a few months. -Search for timeouts in the Apache error log to see if you have timeout issues. -Now that I have explained the general context for any system, -I'll describe concrete examples on several systems, -Debian/Ubuntu and Centos. Feedbacks show that the Centos -process is easier in case you don't know very much any of the -Linux distributions. I add that if you don't know very much any -of the Linux distributions then you shouldn't install this -imapsync service at all. +I use "Timeout 3600", 3600 seconds, an hour. I chose this huge timeout +value because imapsync can spend a long time without talking while +getting the headers of huge folders of 100k messages. If you intend +to offer this service for huge mailboxes or for a long time, I +strongly recommand you to set this "Timeout 3600" in the Apache +configuration right now because you will sure end up with this timeout +issue in a few months. You can search for timeouts in the Apache error +log to see if you have timeout issues. + +Now that I have explained the general context for any system, I'll +describe concrete examples on several systems, Debian/Ubuntu and +Centos. Feedbacks show that the Centos process is easier in case you +don't know very much any Linux distribution. But I have to add that if +you don't know very much the Linux distribution you use, then you +shouldn't install this imapsync service at all. ============================================================================= @@ -175,7 +186,18 @@ ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/ Require all granted -That's all for the Apache configuration side. +In recent Debian distributions you can activate this cgi +stuff with the following commands: + + a2enmod cgi + a2enconf serve-cgi-bin + /etc/init.d/apache2 reload + +If the cgi mode and the cgi-bin configuration are not activated then +you may encounter a 404 error when, later, you will run the command +wget -nv -S -O- http://localhost/cgi-bin/imapsync?testslive=1 + +That's all for the Apache Debian family configuration side. Now get, test, and install the latest imapsync: @@ -189,33 +211,38 @@ Now get, test, and install the latest imapsync: cp imapsync /usr/lib/cgi-bin/ -Assuming that the Unix account running Apache is www-data, -check that it will work under Apache with this command: +Assuming that the Unix account running Apache is www-data, check that +it will work under Apache with this command: su -s /bin/sh -c 'SERVER_SOFTWARE=foo /usr/lib/cgi-bin/imapsync' www-data + +Test that imapsync is considered a cgi with: + wget -nv -S -O- http://localhost/cgi-bin/imapsync?testslive=1 The last command should print something like: Status: 200 OK to sync IMAP boxes. Load on bar is ... ... +If you get a 404 here then review the cgi installation and +configuration part. + You can also verify that the webserver is not buffering its output with the command: wget -nv -S -O- 'http://localhost/cgi-bin/imapsync?testslive=1&simulong=10' -You should get the output as time goes on. If you don't get the -output as time goes on, ie you see no output then all output -at once, it means the webserver is buffering. Fix it with -the "SetEnv no-gzip 1" described above. +You should get the output as time goes on. If you don't get the output +as time goes on, ie you see no output then all output at once, it +means the webserver is buffering. Fix it with the "SetEnv no-gzip 1" +described above. -The UI front-end file place on the server disk in -this example is +The UI front-end file place on the server disk in this example is /var/www/html/X/imapsync_form_extra.html -but it can be placed anywhere on the disk, the important -thing is that it has to be served by the webserver. +but it can be placed anywhere on the disk, the important thing is that +it has to be served by the webserver. mkdir /var/www/html/X/ cd /var/www/html/X/ @@ -228,9 +255,8 @@ thing is that it has to be served by the webserver. The imapsync process working directory in cgi mode is /var/tmp/imapsync_cgi/ -it is not configurable unless changing it in -imapsync directly, it is hard-coded in imapsync. -In this directory will go the log files and +it is not configurable unless changing it in imapsync directly, it is +hard-coded in imapsync. In this directory will go the log files and the pid files. Check @@ -238,14 +264,14 @@ Check or the safer https://yourhost/X/imapsync_form_extra.html -That's all for installing a /X service on Debian. +That's all for installing a /X service on a Debian family system. ============================================================================= B) Here is a concrete example on a Centos 7 server with the Apache webserver httpd: -First, follow and apply the section "Centos 7 and latest imapsync" -at https://imapsync.lamiral.info/INSTALL.d/INSTALL.Centos.txt +First, follow and apply the section "Centos 7 and latest imapsync" at +https://imapsync.lamiral.info/INSTALL.d/INSTALL.Centos.txt Then: @@ -291,14 +317,14 @@ The last command should print something like: Status: 200 OK to sync IMAP boxes. Load on bar is ... ... -You can also verify that the webserver is not buffering its output with the -command: +You can also verify that the webserver is not buffering its output +with the command: wget -nv -S -O- 'http://localhost/cgi-bin/imapsync?testslive=1&simulong=10' -You should get the output as time goes on. If you don't, no output then all -output at once, it means the webserver is buffering. Fix it with -the "SetEnv no-gzip 1" described above. +You should get the output as time goes on. If you don't, no output +then all output at once, it means the webserver is buffering. Fix it +with the "SetEnv no-gzip 1" described above. Now check http://yourhost/X/imapsync_form_extra.html @@ -310,17 +336,19 @@ That's all for installing a /X service on Centos 7. B bis) How about Centos 8? -Follow the procedure for Centos 7. While imapsync is ok on the command line, -you will encounter some permission denied in the CGI context. Something like: +Follow the procedure for Centos 7. While imapsync is ok on the command +line, you will encounter some permission denied in the CGI +context. Something like: wget -nv -S -O- http://localhost/cgi-bin/imapsync?testslive=1 ... -Host1 failure: can not open imap connection on host1 [test1.lamiral.info] -with user [test1]: Unable to connect to test1.lamiral.info: Permission denied +Host1 failure: can not open imap connection on host1 +[test1.lamiral.info] with user [test1]: Unable to connect to +test1.lamiral.info: Permission denied -The issue might come from SELinux. I haven't dig into SELinux enough to give -you the commands that will allow imapsync online and only it while maintaining -SELinux in enforcing mode. +The issue might come from SELinux. I haven't dig into SELinux enough +to give you the commands that will allow imapsync online and only it +while maintaining SELinux in enforcing mode. Quick solution: @@ -336,36 +364,21 @@ To go back to the previous state: Nota bene ========= -You may also want to avoid being placed by systemd in -a directory like (where xxx are crypto hash characters): + +You may also want to avoid being placed by systemd in a directory like +(where xxx are crypto hash characters): /var/tmp/systemd-private-xxx-httpd.service-xxx/tmp/ -In that case, edit the file +In that case, see the Troubleshooting section below. - /usr/lib/systemd/system/httpd.service - -and in it, replace the line - - PrivateTmp=true - -by the line - - PrivateTmp=false - -then reload systemd and restart httpd with the commands: - - systemctl daemon-reload - systemctl restart httpd - systemctl status httpd - ======================================================================= =================== Bandwidth statistics ========================== ======================================================================= -If you want the bandwidth statistics like the ones at the bottom -of the page and following the image link, more detailed at +If you want the bandwidth statistics like the ones at the bottom of +the page and following the image link, more detailed at https://imapsync.lamiral.info/vnstat/vnstati.html Those stats are generated by vnstat @@ -392,39 +405,50 @@ vnstati -5 -o /var/www/vnstat/vnstat_5.png The log says the temporary directory is /var/tmp/imapsync_cgi/ -but it is not in the system. +but this directory is not in the system. What a mystery! -It may be that the apache or httpd service is run by systemd with a jailed -temporary directory. +It may be that the apache or httpd service is run by systemd with a +jailed temporary directory. Solution: - find /etc/systemd/ /usr/lib/systemd/ |xargs grep -s PrivateTmp + find /etc/systemd/ /usr/lib/systemd/ | xargs grep -s PrivateTmp -If systemd jails Apache then you'll find a line like: +If systemd jails Apache, then you'll find a line like: /etc/systemd/system/multi-user.target.wants/apache2.service:PrivateTmp=true (Debian/Ubuntu) or /usr/lib/systemd/system/httpd.service:PrivateTmp=true (Centos) -Replace this line "PrivateTmp=true" -in /etc/systemd/system/multi-user.target.wants/apache2.service +The goal is to override the line + +PrivateTmp=true + +found in /etc/systemd/system/multi-user.target.wants/apache2.service or /usr/lib/systemd/system/httpd.service -with: +with the line: PrivateTmp=false -Then reload the systemd daemon and restart the apache2 service +The right way to do it is by using the "systemctl edit ..." command +and then reload the systemd daemon and restart the apache2 service. +You can also edit directly the file override.conf if you know where +to do it. If you don't use the override.conf mechanism then your change will +be canceled the next time the apache package is updated. Debian: + systemctl edit apache2 + cat /etc/systemd/system/apache2.service.d/override.conf systemctl daemon-reload systemctl restart apache2 systemctl status apache2 Centos: + systemctl edit httpd + cat /etc/systemd/system/httpd.service.d/override.conf systemctl daemon-reload systemctl restart httpd systemctl status httpd @@ -461,6 +485,7 @@ So if your https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js is not what it should be, your access looks compromised. Thanks to Dominik Ulrich for this insight! + ======================================================================= ======================================================================= @@ -487,11 +512,11 @@ Debian: chmod +x imapsync_shell_wrapper cp imapsync_shell_wrapper /usr/lib/cgi-bin/ -Normally, you only have to change the script imapsync_shell_wrapper -to suit your needs. +Normally, you only have to change the script imapsync_shell_wrapper to +suit your needs. -Have in mind that the abort button will kill only one imapsync so -it is not a working button in case of successive imapsync runs. +Have in mind that the abort button will kill only one imapsync so it +is not a working button in case of successive imapsync runs. ======================================================================= ======================================================================= @@ -500,13 +525,12 @@ it is not a working button in case of successive imapsync runs. ====== mod_perl failure ====== This part is for mod_perl experts only. -The script imapsync doesn't work under Modperl::Registry -nor under ModPerl::PerlRun. So read on if you think you -are better than me. -I tried the standard way, telling how any cgi Perl script -can be run under mod_perl perlrun, but it fails with imapsync. -Any hint welcome! +The script imapsync doesn't work under Modperl::Registry nor under +ModPerl::PerlRun. So read on if you think you are better than me. + +I tried the standard way, telling how any cgi Perl script can be run +under mod_perl perlrun, but it fails with imapsync. Any hint welcome! # This is a Debian example @@ -545,6 +569,5 @@ Any hint welcome! curl http://localhost/perl-run/imapsync - ======================================================================= ======================================================================= diff --git a/INSTALL.d/memo_docker b/INSTALL.d/memo_docker index a369ae2..84d5528 100644 --- a/INSTALL.d/memo_docker +++ b/INSTALL.d/memo_docker @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: memo_docker,v 1.15 2021/11/20 20:52:55 gilles Exp gilles $ +# $Id: memo_docker,v 1.17 2022/04/04 14:46:21 gilles Exp gilles $ echo imapsync_docker_timestamp_dockerfile imapsync_docker_timestamp_dockerfile() @@ -16,11 +16,6 @@ imapsync_docker_build() { docker build -t gilleslamiral/imapsync . \ && imapsync_docker_add_tag_version && echo Build SUCCEEDED || echo Build FAILED - - echo - #docker run gilleslamiral/imapsync imapsync --testslive - #docker run gilleslamiral/imapsync imapsync --testslive6 - # docker run gilleslamiral/imapsync imapsync --testslive6 --nossl2 } echo imapsync_docker_version @@ -32,7 +27,7 @@ imapsync_docker_version() echo imapsync_local_docker_version imapsync_local_docker_version() { - docker run gilleslamiral/imapsync imapsync_local --version + docker run gilleslamiral/imapsync /imapsync --version } echo imapsync_docker_add_tag_version @@ -51,15 +46,15 @@ imapsync_docker_add_tag_version() echo imapsync_docker_testslive imapsync_docker_testslive() { - echo docker run gilleslamiral/imapsync imapsync --testslive - docker run gilleslamiral/imapsync imapsync --testslive + echo docker run gilleslamiral/imapsync imapsync --testslive "$@" + docker run gilleslamiral/imapsync imapsync --testslive "$@" } echo imapsync_docker_testslive6 imapsync_docker_testslive6() { - echo docker run gilleslamiral/imapsync imapsync --testslive6 --ssl1 --ssl2 - docker run gilleslamiral/imapsync imapsync --testslive6 --ssl1 --ssl2 + echo docker run gilleslamiral/imapsync imapsync --testslive6 "$@" + docker run gilleslamiral/imapsync imapsync --testslive6 "$@" } echo imapsync_docker_tests @@ -121,13 +116,18 @@ echo imapsync_docker_upload imapsync_docker_upload() { version=`imapsync_docker_version` + version_local=`imapsync_local_docker_version` + # upload only after successful tests set -x \ && is_release_number "$version" \ + && is_release_number "$version_local" \ + && test "$version" = "$version_local" \ && rcsdiff imapsync Dockerfile memo_docker \ && imapsync_docker_testslive \ && imapsync_docker_tests \ && docker run gilleslamiral/imapsync true \ + && docker run gilleslamiral/imapsync diff /imapsync /usr/bin/imapsync \ && docker login --username=gilleslamiral --password=`cat $HOME/var/pass/secret.docker` \ && docker push gilleslamiral/imapsync:latest \ && docker push gilleslamiral/imapsync:$version \ diff --git a/Makefile b/Makefile index f71d3b8..d1e9edf 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -# $Id: Makefile,v 1.339 2022/01/14 14:28:38 gilles Exp gilles $ +# $Id: Makefile,v 1.345 2022/04/06 10:00:39 gilles Exp gilles $ .PHONY: help usage all doc @@ -194,7 +194,6 @@ docker_build: docker_copy_to_vp3 ssh vp3 'cd docker/imapsync && . memo_docker && imapsync_docker_build' - docker_copy_to_vp3: ssh vp3 'mkdir -p docker/imapsync/ var/pass/' rsync -av /g/var/pass/secret.docker vp3:var/pass/secret.docker @@ -287,10 +286,10 @@ testv: imapsync tests.sh tests: test -test: .test_3xx +test: .tests_passed -# .test_3xx is created by tests.sh with success at all mandatory tests -.test_3xx: imapsync tests.sh +# .tests_passed is created by tests.sh with success at all mandatory tests +.tests_passed: imapsync /usr/bin/time sh tests.sh 1>/dev/null unitests: @@ -467,6 +466,20 @@ win64testsdebug: ./W/check_win64err test_testsdebug.bat + +win64testsdebug_p26: + unix2dos W/test_testsdebug.bat + scp imapsync W/test_testsdebug.bat gille@p26:'Desktop\imapsync_build' + ssh gille@p26 'Desktop\imapsync_build\test_testsdebug.bat' + ./W/check_p26err test_testsdebug.bat + +win64tests_p26: + unix2dos W/test_tests.bat + scp imapsync W/test_tests.bat gille@p26:'Desktop\imapsync_build' + ssh gille@p26 'Desktop\imapsync_build\test_tests.bat' + ./W/check_p26err test_tests.bat + + zzz: unix2dos W/build_exe.bat W/install_module_one.bat scp imapsync W/build_exe.bat W/install_module_one.bat W/test_exe_testsdebug.bat pc_HP_DV7_p24:'Desktop/imapsync_build' @@ -535,12 +548,18 @@ win64build: winprepalocal ssh 'pc HP DV7'@p24 'Desktop/imapsync_build/build_exe.bat' ./W/check_win64err build_exe.bat -imapsync.exe: imapsync_64bit.exe +imapsync.exe: imapsync_64bit.exe_p26 cp -a imapsync_64bit.exe imapsync.exe -imapsync_64bit.exe: imapsync winprepalocal + + +.PHONY: imapsync_64bit.exe_p24 imapsync_64bit.exe_p26 + + +imapsync_64bit.exe_p24: imapsync (date "+%s"| tr "\n" " "; echo -n "BEGIN 64bit " $(VERSION) ": "; date) >> W/.BUILD_EXE_TIME + unix2dos W/build_exe.bat W/test_exe.bat W/install_modules.bat W/install_module_one.bat ssh 'pc HP DV7'@p24 'perl -V' scp imapsync W/build_exe.bat W/install_modules.bat W/install_module_one.bat \ W/test_exe_tests.bat W/test_exe_testsdebug.bat W/test_exe.bat \ @@ -554,13 +573,31 @@ imapsync_64bit.exe: imapsync winprepalocal chmod a+r+x imapsync_64bit.exe (date "+%s"| tr "\n" " "; echo -n "END 64bit " $(VERSION) ": "; date) >> W/.BUILD_EXE_TIME +imapsync_64bit.exe: imapsync + (date "+%s"| tr "\n" " "; echo -n "BEGIN 64bit " $(VERSION) ": "; date) >> W/.BUILD_EXE_TIME + unix2dos W/build_exe.bat W/test_exe.bat W/install_modules.bat W/install_module_one.bat + ssh gille@p26 'perl -V' + ssh gille@p26 'if not exist Desktop\imapsync_build mkdir Desktop\imapsync_build' + scp imapsync W/build_exe.bat W/install_modules.bat W/install_module_one.bat \ + W/test_exe_tests.bat W/test_exe_testsdebug.bat W/test_exe.bat \ + gille@p26:'Desktop\imapsync_build' + ssh gille@p26 'Desktop\imapsync_build\build_exe.bat' + ./W/check_p26err build_exe.bat + scp ../../var/pass/secret.tata ../../var/pass/secret.titi gille@p26:'Desktop\imapsync_build' + ssh gille@p26 'Desktop\imapsync_build\test_exe.bat' + ./W/check_p26err test_exe.bat + rm -f imapsync_64bit.exe + scp -T gille@p26:'Desktop\imapsync_build\imapsync_64bit.exe' . + chmod a+r+x imapsync_64bit.exe + (date "+%s"| tr "\n" " "; echo -n "END 64bit " $(VERSION) ": "; date) >> W/.BUILD_EXE_TIME + zip: dosify_bat rm -rfv ../prepa_zip/imapsync_$(VERSION)/ mkdir -p ../prepa_zip/imapsync_$(VERSION)/FAQ.d/ ../prepa_zip/imapsync_$(VERSION)/Cook/ cp -av examples/imapsync_example.bat examples/sync_loop_windows.bat examples/file.txt ../prepa_zip/imapsync_$(VERSION)/ cp -av W/build_exe.bat W/install_modules.bat W/test_cook_exe.bat W/test_cook_src.bat imapsync ../prepa_zip/imapsync_$(VERSION)/Cook/ - for f in README ; do cp -av $$f ../prepa_zip/imapsync_$(VERSION)/$$f.txt ; done + cp -av README.txt ../prepa_zip/imapsync_$(VERSION)/ cp -av FAQ.d/*.txt ../prepa_zip/imapsync_$(VERSION)/FAQ.d/ cp -av imapsync.exe imapsync_32bit.exe README_Windows.txt ../prepa_zip/imapsync_$(VERSION)/ unix2dos ../prepa_zip/imapsync_$(VERSION)/*.txt @@ -572,7 +609,7 @@ zip: dosify_bat # C:\Users\mansour\Desktop\imapsync -.PHONY: mac maccopy macforce mactests mactestsdebug mactestslive mactestslive6 bin win lin win64 +.PHONY: mac macstadiumcopy maccopy macforce mactests mactestsdebug mactestslive mactestslive6 bin win lin win64 mac: imapsync_bin_Darwin @@ -590,7 +627,7 @@ maccopy: macforce: maccopy ssh -4 -p 995 gilleslamira@gate.polarhome.com 'sh -x build_mac.sh' -imapsync_bin_Darwin: imapsync W/build_mac.sh INSTALL.d/prerequisites_imapsync maccopy +imapsync_bin_Darwin: maccopy imapsync W/build_mac.sh INSTALL.d/prerequisites_imapsync rcsdiff imapsync ssh -4 -p 995 gilleslamira@gate.polarhome.com 'sh -x build_mac.sh' rsync -P -e 'ssh -4 -p 995' gilleslamira@gate.polarhome.com:imapsync_bin_Darwin . @@ -617,7 +654,7 @@ bin: mac win lin: $(BIN_NAME) -win: win64 win32 imapsync.exe +win: win32 win64 imapsync.exe win32: imapsync_32bit.exe @@ -709,10 +746,10 @@ biz: S/imapsync_sold_by_country.txt docker_pull_count auto_ci: docker_pull_count docker_pull_count: - cd W/ && rcsdiff docker_pull_count.txt || { echo | ci -l docker_pull_count.txt ; } + rcsdiff W/docker_pull_count.txt || { echo | ci -l W/docker_pull_count.txt ; } S/imapsync_sold_by_country.txt: /g/bin/imapsync_by_country - cd S/ && /g/bin/imapsync_by_country && echo | ci -l imapsync_sold_by_country.txt + cd S/ && /g/bin/imapsync_by_country && { echo | ci -l imapsync_sold_by_country.txt ; } ks: @@ -828,7 +865,7 @@ W/.valid.index.shtml: index.shtml S/*.shtml S/template_xhtml1.shtml touch W/.valid.index.shtml -.PHONY: upload_index upload_FAQ ci_imapsync upload_bin +.PHONY: upload_index ci_imapsync upload_latest upload_FAQ upload_bin upload_index: valid_index clean_permissions @@ -855,16 +892,65 @@ upload_latest: unitests ci_imapsync bin rsync -aHvzP --delete ../imapsync_website/ root@ks5.lamiral.info:/usr/local/www/apache24/data/imapsync/ -upload_cgi: unitests ks5tests ks5tests_root ci_imapsync - rsync -a imapsync ./INSTALL.d/prerequisites_imapsync ../imapsync_website/ - rsync -aHvz --delete ../imapsync_website/ root@ks5.lamiral.info:/usr/local/www/apache24/data/imapsync/ - rsync -P imapsync root@ks5.lamiral.info:/home/www/apache24/cgi-bin/ + +.PHONY: upload_cgi upload_cgi_ks5 upload_cgi_memo upload_cgi_vp3 upload_cgi_vp4 upload_cgi_ks6 + +upload_cgi: upload_cgi_ks5 upload_cgi_vp3 upload_cgi_vp4 upload_cgi_ks6 + +upload_cgi_ks5: ci_imapsync unitests ks5tests + rsync -P imapsync root@ks5.lamiral.info:/home/www/apache24/cgi-bin/imapsync_new + curl -v --data 'testslive=1' https://imapsync.lamiral.info/cgi-bin/imapsync_new | grep 'Exiting with return value 0' + rsync -P imapsync root@ks5.lamiral.info:/home/www/apache24/cgi-bin/imapsync + curl -v --data 'testslive=1' https://imapsync.lamiral.info/cgi-bin/imapsync | grep 'Exiting with return value 0' upload_cgi_memo: dos2unix X/stat_patterns.txt X/server_survey_patterns.txt sed -i".bak" '/^[[:space:]]*$$/d' X/stat_patterns.txt X/server_survey_patterns.txt rsync -av X/cgi_memo X/stat_patterns.txt X/server_survey_patterns.txt root@ks5:/var/tmp/imapsync_cgi/ +upload_cgi_memo_all: + rsync -av X/cgi_memo root@ks5:/var/tmp/imapsync_cgi/ + rsync -av X/cgi_memo root@ks6:/var/tmp/imapsync_cgi/ + rsync -av X/cgi_memo root@vp3:/var/tmp/imapsync_cgi/ + rsync -av X/cgi_memo root@vp4:/var/tmp/imapsync_cgi/ + +# Debian +upload_cgi_vp3: ci_imapsync vp3tests + rsync -P imapsync root@vp3.lamiral.info:/usr/lib/cgi-bin/imapsync_new + curl -v --data 'testslive=1' https://vp3.lamiral.info/cgi-bin/imapsync_new | grep 'Exiting with return value 0' + rsync -P imapsync root@vp3.lamiral.info:/usr/lib/cgi-bin/imapsync + curl -v --data 'testslive=1' https://vp3.lamiral.info/cgi-bin/imapsync | grep 'Exiting with return value 0' + +# Centos +upload_cgi_vp4: ci_imapsync vp4tests + rsync -P imapsync root@vp4.lamiral.info:/var/www/cgi-bin/imapsync_new + curl -v --data 'testslive=1' https://vp4.lamiral.info/cgi-bin/imapsync_new | grep 'Exiting with return value 0' + rsync -P imapsync root@vp4.lamiral.info:/var/www/cgi-bin/imapsync + curl -v --data 'testslive=1' https://vp4.lamiral.info/cgi-bin/imapsync | grep 'Exiting with return value 0' + +# Debian +upload_cgi_ks6: ci_imapsync ks6tests + rsync -P imapsync root@ks6.lamiral.info:/usr/lib/cgi-bin/imapsync_new + curl -v --data 'testslive=1' https://ks6.lamiral.info/cgi-bin/imapsync_new | grep 'Exiting with return value 0' + rsync -P imapsync root@ks6.lamiral.info:/usr/lib/cgi-bin/imapsync + curl -v --data 'testslive=1' https://ks6.lamiral.info/cgi-bin/imapsync | grep 'Exiting with return value 0' + +.PHONY: vp3tests vp4tests ks6tests + +vp3tests: + rsync -P imapsync root@vp3.lamiral.info:imapsync + ssh root@vp3.lamiral.info ./imapsync --tests + ssh root@vp3.lamiral.info ./imapsync --testslive6 + +vp4tests: + rsync -P imapsync root@vp4.lamiral.info:imapsync + ssh root@vp4.lamiral.info ./imapsync --tests + ssh root@vp4.lamiral.info ./imapsync --testslive6 + +ks6tests: + rsync -P imapsync root@ks6.lamiral.info:imapsync + ssh root@ks6.lamiral.info ./imapsync --tests + ssh root@ks6.lamiral.info ./imapsync --testslive6 upload_X: ./W/tools/validate_xml_html5 X/index.html X/imapsync_form.html X/imapsync_form_extra.html X/imapsync_form_extra_free.html X/imapsync_form_wrapper.html diff --git a/README b/README index 12fce91..e463c5c 100644 --- a/README +++ b/README @@ -6,7 +6,7 @@ NAME VERSION - This documentation refers to Imapsync $Revision: 2.178 $ + This documentation refers to Imapsync $Revision: 2.200 $ USAGE @@ -38,7 +38,7 @@ DESCRIPTION that are on the destination side but not on the source side stay as they are. See the --delete2 option to have strict sync and delete them. - How imapsync know a message is already on both sides? Same specific + How does imapsync know a message is already on both sides? Same specific headers and the transfer is done only once. By default, the identification headers are "Message-Id:" and "Received:" lines but this choice can be changed with the --useheader option, most often a @@ -46,8 +46,8 @@ DESCRIPTION All flags are preserved, unread messages will stay unread, read ones will stay read, deleted will stay deleted. In the IMAP protocol, a - deleted message is not really deleted, it is marked \Deleted and can be - undelete. Real destruction comes with the EXPUNGE or UIDEXPUNGE IMAP + deleted message is not deleted, it is marked \Deleted and can be + undeleted. Real destruction comes with the EXPUNGE or UIDEXPUNGE IMAP commands. You can abort the transfer at any time and restart it later, imapsync @@ -76,16 +76,35 @@ DESCRIPTION There is no unidentified message on host1. Detected 0 errors + Imapsync mentions the total sizes of both accounts at the beginning of + the sync and also at the end. Sometimes, even with a strict sync, those + total sizes differ, and sometimes they differ a lot. The difference is + not a good criterion to conclude the sync went wrong. + + Why? That's because message sizes given by the imap servers are not + always accurate, they are not always the same as the actual message + sizes of the messages transferred by imapsync. Imapsync use the sizes + given by the imap servers to calculate the big total size. They can + differ. In the early days, Imapsync used the sizes of the messages as + one of the criteria to identify the messages, different sizes implied + different messages; but it was a mistake, the same message had different + sizes on both sides sometimes, depending on the imap servers. + + Another explanation for a big total size difference is that Gmail + doesn't count the size of duplicate messages across folders twice, while + imapsync does. + A classical scenario is synchronizing a mailbox B from another mailbox A where you just want to keep a strict copy of A in B. Strict meaning all messages in A will be in B but no more. - For this, option --delete2 can be used, it deletes messages in the host2 - folder B that are not in the host1 folder A. If you also need to destroy - host2 folders that are not in host1 then use --delete2folders. See also - --delete2foldersonly and --delete2foldersbutnot to set up exceptions on - folders to destroy. INBOX will never be destroyed, it's a mandatory - folder in IMAP so imapsync doesn't even try to remove it. + For a strict synchronization, use the option --delete2. The option + --delete2 deletes the messages in the host2 folder B that are not in the + host1 folder A. If you also need to destroy host2 folders that are not + in host1 then use --delete2folders. See also --delete2foldersonly and + --delete2foldersbutnot to set up exceptions on folders to destroy. INBOX + will never be destroyed, it's a mandatory folder in IMAP so imapsync + doesn't even try to remove it. A different scenario is to delete the messages from the source mailbox after a successful transfer, it can be a good feature when migrating @@ -114,18 +133,19 @@ OPTIONS usage: imapsync [options] The standard options are the six values forming the credentials. Three - values on each side are needed in order to login into the IMAP servers. - These six values are a hostname, a username, and a password, two times. + values on each side are needed to login into the IMAP servers. These six + values are a hostname, a username, and a password, two times. - Conventions used in the following descriptions of the options: + Here are the conventions used in the following descriptions of the + options: - str means string - int means integer number - flo means float number - reg means regular expression - cmd means command + str means a string + int means an integer number + flo means a float number + reg means a regular expression + cmd means a command - --dry : Makes imapsync doing nothing for real; it just print what + --dry : Makes imapsync do nothing for real; it just prints what would be done without --dry. OPTIONS/credentials @@ -237,9 +257,9 @@ OPTIONS --nomixfolders : Do not merge folders when host1 is case-sensitive while host2 is not (like Exchange). Only the first - similar folder is synced (example: with folders - "Sent", "SENT" and "sent" - on host1 only "Sent" will be synced to host2). + similar folder is synced. Example: with folders + "Sent", "SENT" and "sent" on host1, only "Sent" + will be synced to host2. --skipemptyfolders : Empty host1 folders are not created on host2. @@ -726,8 +746,8 @@ LICENSE AND COPYRIGHT Imapsync is free, open, public but not always gratis software cover by the NOLIMIT Public License, now called NLPL. See the LICENSE file - included in the distribution or just read this simple sentence as it IS - the licence text: + included in the distribution or just read the following simple sentence + as it IS the license text: "No limits to do anything with this work and this license." @@ -744,7 +764,7 @@ AUTHOR Good feedback is always welcome. Bad feedback is very often welcome. Gilles LAMIRAL earns his living by writing, installing, configuring and - sometimes teaching free, open and often gratis software. Imapsync used + sometimes teaching free, open, and often gratis software. Imapsync used to be "always gratis" but now it is only "often gratis" because imapsync is sold by its author, your servitor, a good way to maintain and support free open public software tools over decades. @@ -796,19 +816,19 @@ INSTALL Imapsync works under most Windows (2000, XP, Vista, Seven, Eight, Ten and all Server releases 2000, 2003, 2008 and R2, 2012 and R2, 2016) - as a standalone binary software called imapsync.exe, - usually launched from a batch file in order to avoid always typing - the options. There is also a 32bit binary called imapsync_32bit.exe + as a standalone binary software called imapsync.exe, usually launched + from a batch file in order to avoid always typing the options. There + is also a 32bit binary called imapsync_32bit.exe - Imapsync works under OS X as a standalone binary - software called imapsync_bin_Darwin + Imapsync works under OS X as a standalone binary software called + imapsync_bin_Darwin Purchase latest imapsync at https://imapsync.lamiral.info/ - You'll receive a link to a compressed tarball called imapsync-x.xx.tgz - where x.xx is the version number. Untar the tarball where - you want (on Unix): + You'll receive a link to a compressed tarball called + imapsync-x.xx.tgz where x.xx is the version number. + Untar the tarball where you want (on Unix): tar xzvf imapsync-x.xx.tgz diff --git a/README_Windows.txt b/README_Windows.txt index b51162a..e6bd0ca 100644 --- a/README_Windows.txt +++ b/README_Windows.txt @@ -1,4 +1,4 @@ -# $Id: README_Windows.txt,v 1.26 2022/01/14 11:54:28 gilles Exp gilles $ +# $Id: README_Windows.txt,v 1.27 2022/04/05 19:45:21 gilles Exp gilles $ # # This is the README_Windows.txt file for imapsync # imapsync: IMAP syncing and migration tool. @@ -43,13 +43,13 @@ A.0) Preamble for visual users looking for a visual tool. A.1) Get imapsync. Get imapsync at https://imapsync.lamiral.info/ - You'll then have access to a zip archive file named imapsync_2.178.zip - where 2.178 is the imapsync release number. + You'll then have access to a zip archive file named imapsync_2.200.zip + where 2.200 is the imapsync release number. A.2) Extract the zip file in a folder where you will work with imapsync. You can work on the Desktop since the zip file extraction creates - a unique folder named imapsync_2.178/ + a unique folder named imapsync_2.200/ Do not unzip the archive in what is called a "system" directory since you may encounter permission issues. @@ -62,7 +62,7 @@ In short, any user on your system can use imapsync. A.3) Check the folder - In the folder extracted and called imapsync_2.178, you see 7 files + In the folder extracted and called imapsync_2.200, you see 7 files and 2 directories. Those files and directories may be presented in a different order than the following, the order is not important anyway. There are only two important files to get your mailbox diff --git a/S/external.shtml b/S/external.shtml index 697c6fd..d3b10f0 100644 --- a/S/external.shtml +++ b/S/external.shtml @@ -91,12 +91,11 @@

These services are similar to the -imapsync online service +imapsync online service I call /X. Some are an exact copy of /X, some are not, some use Imapsync, some don't.
I have no share in these free or paid services. -Prices are given par mailbox and may be outdated -(Last checked on Tuesday 10th November 2020). +Prices are given par mailbox and may be outdated (Last checked on Saturday 19 February 2022).

Imapsync based

@@ -122,29 +121,28 @@ this is a better choice to avoid duplicates with some imap servers.

    +
  1. NameHero 0 USD: https://imapsync.net/ (imapsync 2.140)
  2. Slovenia Control Panel 0 USD: https://tools.controlpanel.si/imapsync/ (imapsync 1.810)
  3. Intertune Cloud 0 USD: https://tools.intertune.io/imapsync/X/ (imapsync 1.882)
  4. Web Hosting Canada 0 USD: https://imapsync.whc.ca/ (imapsync 1.882)
  5. BoomHost 0 USD: https://imapsync.boomhost.com/ (imapsync 1.925)
    6. -
  6. WebHosting|4u (greek) 0 USD: https://imapcopy.webhosting4u.gr/ (imapsync 1.977)
    7. +
  7. WebHosting|4u (greek) 0 USD: https://imapcopy.webhosting4u.gr/ (imapsync 2.178)
  8. Vimexx 0 USD: https://imapsync.nl/ (imapsync 1.991)
    9. -
  9. Timetakernet 0 USD: https://mailsync.timetakernet.info/ (imapsync 1.998)
  10. KeurigOnline 0 USD: https://imapsync.keurigonline.nl/ (imapsync 1.998)
    11. - -
  11. UK Migration Wizard 0 USD: https://app.migrationwizard.co.uk/ (imapsync ?.???)

Not Imapsync based

  1. Ovh migration 0 EUR: https://omm.ovh.net/Migration/Create
  2. Rackspace migration 0 USD: http://www.rackspace.com/email-hosting/migrations
    3. -
  3. Movemymail one free, 5 USD after: https://movemymail.net/
    4. -
  4. TransferMyEmail 1.50 USD for a day pass: https://transfermyemail.ca/
    5. -
  5. Migrationwiz 11.99 USD: https://www.bittitan.com/products/migrationwiz/
    6. -
  6. Audriga 11.90 EUR: https://www.groupware-migration.com/
    7. +
  7. Movemymail one free, 5 USD after: https://movemymail.net/
    8. +
  8. TransferMyEmail 1.50 USD for a day pass: https://transfermyemail.ca/
    9. +
  9. Migrationwiz 12 USD: https://www.bittitan.com/products/migrationwiz/
    10. +
  10. Audriga 11.90 EUR: https://www.audriga.com/
  11. Yippiemove 14.95 USD (discontinued on April 2019): http://www.yippiemove.com/ ( See Yippiemove vs ImapSync )
  12. ShuttleCloud one free, 19.95 USD after https://www.shuttlecloud.com/
    13. +
  13. Cloudasta ??? USD: https://cloudasta.com/

Imap to files services

@@ -186,7 +184,7 @@ alt="Viewable With Any Browser" /> This document last modified on -($Id: external.shtml,v 1.59 2022/01/08 19:54:26 gilles Exp gilles $)
+($Id: external.shtml,v 1.61 2022/02/19 20:26:25 gilles Exp gilles $)
Top of the page

diff --git a/S/images/logo_imapsync_s1.png b/S/images/logo_imapsync_s1.png new file mode 100644 index 0000000000000000000000000000000000000000..2526144a019763944e6ecf5c8b9a9a00cea589dc GIT binary patch literal 3957 zcmai1hc_Eu^bd)>Rcmi;gV@xlQd_Ku){a@Dc2pu(&C*s;<4cQLQM+jEQKS?lR6_|> zqSW5AR{5pB-yiTh@7{OMdFOr39q--y-hFOvX23+xLk|D|n4pHb7G$YT<{PxsNt);M~QB)6SN0SMn}`z%x;%@Wu%IO%DrYLoCQ)Sa?1jfA$S`Z`_X~2v6oG8(K<714+=IKuvQa^@0g^~M14=6~n{^_)4XFDQ znvLYv9&38Z)xz@yka1UXhcA_>(205;$0bV@am9w_T&_@z-H*200RLD5CoIJ8yH~KI zluPYM;San;5dgRZMo@r(&^c#73*~EyS7F2r#4K{_?Vi2eZ@fwk-pCzEjry zapz(`+~d1c#{uVG&R$V`=ZtZgqndMy)&ycdA$Ckdly(GD2N||NkrK4A%b~M3=@Fm{+7+A(y8zSHrTnq zIrm7Mfhq8RbDH98TEWc<9N9M}1qo_J#1Pc>lTODv0cNc_(93lAw zvmLjiL}=&NEWCOhN-Rk3d3l}(jg41D+61UT>Lb1T5u5z{6(ViP#CLs*9qSE}#jBXO%|dh0|dY^@!qJ5RMpKAKf0qc0T`W8^EAhqRu#Z4974b z`H{3XPPNuVY5SuJpPM(B-#N4&mHO0A`HP9G*P31~00gkd{xbXOF;^w6cc3r*8Ow)ZzA2|?W z=1f(a=vt1DsNVUiy1Zs@UEQ{Zj}@w^v?}|aDOoGVXN7cNA)*0K4Rx{Y?eWMKXvwH9 z$90KJx*yZ`P$-lqqGH;%+t=P%Seerr z`M1MqYt2qYU8Us=2#hP>44{ZiaH2k%;{<6VJI=~mStzY3c-z`vF;rLc|Cv7e9@>uS zlD{WWZDR+9(A_q>(jOQ&LtT`rBT)i}G~ENc$(hDwXG_Gz>1<3Z4;~*g9=1gbfuCsK zz=Dxj)OuU$2!>AiCvq8`h~?qE6|tSt`{L}V^K|Wqh4CW|>D6t@F-PFXgaYL=ciZr4 zLirs~%X8geCm4)$x9I(`BsnRhXkaoNBEF=~-YcLd9%J2T&*%N_ZEd3*FI2Gv+RQ;l zvKJdOw}*;!>L-I~p`*0%*e`WlUryp!i18++dF!=Z3y<^&Nld(3?S8{K_}P%Piklj& zNKwEYLCvQZ!jY&8%-n%~aTcNl>0VdmvpRfozWVCzSa0x{i4atnng0h&B6D$2l1K;B z?6_iDzq0M5FTz%AQL!^&4`gKIbAD8pvH<>42UYoL1V$`FoJI z?+cUdq=r)Xi}_}HpJNSMA+j182{p~E&&y+Ar_qge@o8zzsbX|mtY6%Wp7h9o+VG;842GD$(!Iqg7c)M*S0i#{4}EaEL#yH=^^UHWJ{NA%e|8$ zG_)`n89zU$@AUj?S@XOK^ERa4wBEhbn;pca#mWj|8Hv*`P7s5NW$^n>?XfQIae3y4 zLQ`MD`0XswH7)b?cM`F8T2QM~QGjy)bBDgpA3ApjIHCtE2RDR0L-ky{2FfrsK z({vNy+aZ_WL==g^F6*jv#{ovo((-Ta;rg~xQvhi%Rfu+r?||=puE`S0Gy_^jWmF>l zsPz`2d7$&Lj_-6+URs*`aA|Dmb=hI|bPjp|l`b3?$TzqcxOQk;TB@J@hVR#ur(hpC zm;7vq|NXmeZkdstEgu(Gi#%Oj9vyG|?`0e5P(AgnET?0Lo?X`X*;R76_n$e?fzm7S zmxX>7a3J<7Qk|X}B_$>8O+L8I#jTpO?p^Nv#!xI?47$(51}Z03B_%V}d5r~oH*z!c zspv{$cy>b)e4CyuOmfh_5YMz?WhU>~bf0cAFuXBM!Nogmtky9nI=;u|lVQ^l=Jx_y zPO{`ssteuknvua9iK5(Mai4YLlX(UE_fd*cF)j_x(`eMp@0FgkiV8Vu8YhoOkGT3j ztZr)-+?VL7aS%nJAl>^21W`D1O^h!q(1BEjzB4%A`*-Lh6XGMtumPWZhnNzCa~#61 z>M00Bo%jFYyAr4<6<=A(s>jyvC9ilO(H?vU-^+O^03e?}5O}-Xs_e^Rm`&rF1f=T2%Zu$AP zn{{XycQ}4(^32W`SWH&tX=rFj#iNV7ew;694(+I$a~yVzH1L%{OXOoJb4*}@ZyOtd zTC6aU%1+*)z#K&$G}%EizOGWm3pwaFe%f zYY!iAQ#0q_CN4fP?c41;duSt;IT&MLIpcGp8Kbl~Do#N>0=2YVK~A z)wgpb;@ch;~oED@VN@@i#erPQt(xidHIJNuDNj}8uo6xtvRMF?snyH3ILz!(zK z(l=i_MII$`2#Vo7fe>ASPC6(dA%R>HOoJ+{4AS1x)@?Z33j5j@Px;QNd<~TP9hai& zvJ&>hb(fJo-cce`UQ|@)$n*0wQT#vB-$10^l_e#ixCB-|6l;58!)S zzJ6_11c#nJ5k0!fm1Zc=v!iqaFE8}X_s79e4c02x#iMlS-~hC^r**U$9_M{1+ca~R z7UBwb9zPox2qf1LvXf+ESanf%wAE*gUzWJTx*6JH>vO4TS$7j-QQHz(C7}%3|n0Rzjw!%syb^OsLN1&-aoN`^ z?V(=beqzG;HFX!%bAYQ(Bo&RM}Po&Jn?T+hkqscF8seCyB+!&RW05 zyezhFRMnace&EPTR$^k7A}=SWUm_pxKAt8<*Yhmhup!N8(eOt7=Y?wW(I9Q|o%Ri7 zw#tl&hYQAyVXp}U;e@B8ZnV$WA(Pfnlb-TQwN=zjKfg`4dItz1DTw+TSG^&2e5hW@L`g_sSWvcSmZ4StHF;S$uuegDoi?oIwWe1ewz$aQe&aJRMc$+E1P zSg)_ovb>F`9kE(FoT_slHY2V#dQJPAjC(6QrJ;6|gqfubzc~mh%lD3&Il^f1>&ceG7lU?ohf+RG!i zl)E~|qv=bXt72}`_`~m#S(chVAKHvH`I*{URr}gDK7Vv#9vTwj>`JWR#HWnC`K;OS zy6;l8Tp%JRdNWjjMWZEixgaY2?-^!)DOYCqXgd5@bIYT)kM#yaB)KEKj?%M1J$U5q zf^wH~BToQ%Svgs$f24X}RzXcpSxxDdq^!J}tgNc-FLGb|e*nJ#H!qLS{|Df)^2&e= w;QpuKUVsM*Rcmi;gV@xlQd_Ku){a@Dc2pu(&C*s;<4cQLQM+jEQKS?lR6_|> zqSW5AR{5pB-yiTh@7{OMdFOr39q--y-hFOvX23+xLk|D|n4pHb7G$YT<{PxsNt);M~QB)6SN0SMn}`z%x;%@Wu%IO%DrYLoCQ)Sa?1jfA$S`Z`_X~2v6oG8(K<714+=IKuvQa^@0g^~M14=6~n{^_)4XFDQ znvLYv9&38Z)xz@yka1UXhcA_>(205;$0bV@am9w_T&_@z-H*200RLD5CoIJ8yH~KI zluPYM;San;5dgRZMo@r(&^c#73*~EyS7F2r#4K{_?Vi2eZ@fwk-pCzEjry zapz(`+~d1c#{uVG&R$V`=ZtZgqndMy)&ycdA$Ckdly(GD2N||NkrK4A%b~M3=@Fm{+7+A(y8zSHrTnq zIrm7Mfhq8RbDH98TEWc<9N9M}1qo_J#1Pc>lTODv0cNc_(93lAw zvmLjiL}=&NEWCOhN-Rk3d3l}(jg41D+61UT>Lb1T5u5z{6(ViP#CLs*9qSE}#jBXO%|dh0|dY^@!qJ5RMpKAKf0qc0T`W8^EAhqRu#Z4974b z`H{3XPPNuVY5SuJpPM(B-#N4&mHO0A`HP9G*P31~00gkd{xbXOF;^w6cc3r*8Ow)ZzA2|?W z=1f(a=vt1DsNVUiy1Zs@UEQ{Zj}@w^v?}|aDOoGVXN7cNA)*0K4Rx{Y?eWMKXvwH9 z$90KJx*yZ`P$-lqqGH;%+t=P%Seerr z`M1MqYt2qYU8Us=2#hP>44{ZiaH2k%;{<6VJI=~mStzY3c-z`vF;rLc|Cv7e9@>uS zlD{WWZDR+9(A_q>(jOQ&LtT`rBT)i}G~ENc$(hDwXG_Gz>1<3Z4;~*g9=1gbfuCsK zz=Dxj)OuU$2!>AiCvq8`h~?qE6|tSt`{L}V^K|Wqh4CW|>D6t@F-PFXgaYL=ciZr4 zLirs~%X8geCm4)$x9I(`BsnRhXkaoNBEF=~-YcLd9%J2T&*%N_ZEd3*FI2Gv+RQ;l zvKJdOw}*;!>L-I~p`*0%*e`WlUryp!i18++dF!=Z3y<^&Nld(3?S8{K_}P%Piklj& zNKwEYLCvQZ!jY&8%-n%~aTcNl>0VdmvpRfozWVCzSa0x{i4atnng0h&B6D$2l1K;B z?6_iDzq0M5FTz%AQL!^&4`gKIbAD8pvH<>42UYoL1V$`FoJI z?+cUdq=r)Xi}_}HpJNSMA+j182{p~E&&y+Ar_qge@o8zzsbX|mtY6%Wp7h9o+VG;842GD$(!Iqg7c)M*S0i#{4}EaEL#yH=^^UHWJ{NA%e|8$ zG_)`n89zU$@AUj?S@XOK^ERa4wBEhbn;pca#mWj|8Hv*`P7s5NW$^n>?XfQIae3y4 zLQ`MD`0XswH7)b?cM`F8T2QM~QGjy)bBDgpA3ApjIHCtE2RDR0L-ky{2FfrsK z({vNy+aZ_WL==g^F6*jv#{ovo((-Ta;rg~xQvhi%Rfu+r?||=puE`S0Gy_^jWmF>l zsPz`2d7$&Lj_-6+URs*`aA|Dmb=hI|bPjp|l`b3?$TzqcxOQk;TB@J@hVR#ur(hpC zm;7vq|NXmeZkdstEgu(Gi#%Oj9vyG|?`0e5P(AgnET?0Lo?X`X*;R76_n$e?fzm7S zmxX>7a3J<7Qk|X}B_$>8O+L8I#jTpO?p^Nv#!xI?47$(51}Z03B_%V}d5r~oH*z!c zspv{$cy>b)e4CyuOmfh_5YMz?WhU>~bf0cAFuXBM!Nogmtky9nI=;u|lVQ^l=Jx_y zPO{`ssteuknvua9iK5(Mai4YLlX(UE_fd*cF)j_x(`eMp@0FgkiV8Vu8YhoOkGT3j ztZr)-+?VL7aS%nJAl>^21W`D1O^h!q(1BEjzB4%A`*-Lh6XGMtumPWZhnNzCa~#61 z>M00Bo%jFYyAr4<6<=A(s>jyvC9ilO(H?vU-^+O^03e?}5O}-Xs_e^Rm`&rF1f=T2%Zu$AP zn{{XycQ}4(^32W`SWH&tX=rFj#iNV7ew;694(+I$a~yVzH1L%{OXOoJb4*}@ZyOtd zTC6aU%1+*)z#K&$G}%EizOGWm3pwaFe%f zYY!iAQ#0q_CN4fP?c41;duSt;IT&MLIpcGp8Kbl~Do#N>0=2YVK~A z)wgpb;@ch;~oED@VN@@i#erPQt(xidHIJNuDNj}8uo6xtvRMF?snyH3ILz!(zK z(l=i_MII$`2#Vo7fe>ASPC6(dA%R>HOoJ+{4AS1x)@?Z33j5j@Px;QNd<~TP9hai& zvJ&>hb(fJo-cce`UQ|@)$n*0wQT#vB-$10^l_e#ixCB-|6l;58!)S zzJ6_11c#nJ5k0!fm1Zc=v!iqaFE8}X_s79e4c02x#iMlS-~hC^r**U$9_M{1+ca~R z7UBwb9zPox2qf1LvXf+ESanf%wAE*gUzWJTx*6JH>vO4TS$7j-QQHz(C7}%3|n0Rzjw!%syb^OsLN1&-aoN`^ z?V(=beqzG;HFX!%bAYQ(Bo&RM}Po&Jn?T+hkqscF8seCyB+!&RW05 zyezhFRMnace&EPTR$^k7A}=SWUm_pxKAt8<*Yhmhup!N8(eOt7=Y?wW(I9Q|o%Ri7 zw#tl&hYQAyVXp}U;e@B8ZnV$WA(Pfnlb-TQwN=zjKfg`4dItz1DTw+TSG^&2e5hW@L`g_sSWvcSmZ4StHF;S$uuegDoi?oIwWe1ewz$aQe&aJRMc$+E1P zSg)_ovb>F`9kE(FoT_slHY2V#dQJPAjC(6QrJ;6|gqfubzc~mh%lD3&Il^f1>&ceG7lU?ohf+RG!i zl)E~|qv=bXt72}`_`~m#S(chVAKHvH`I*{URr}gDK7Vv#9vTwj>`JWR#HWnC`K;OS zy6;l8Tp%JRdNWjjMWZEixgaY2?-^!)DOYCqXgd5@bIYT)kM#yaB)KEKj?%M1J$U5q zf^wH~BToQ%Svgs$f24X}RzXcpSxxDdq^!J}tgNc-FLGb|e*nJ#H!qLS{|Df)^2&e= w;QpuKUVsM* --> +
    +
  • 2.200
    • +
  • Dependencies removed: None
    • +
  • Dependencies added: None
    • + +
  • Enhancement: Append a final email report on each account at the end of the synchronization. Use --noemailreport1 and --noemailreport2 to avoid final emails reports in each INBOX.
    • +
  • Enhancement: Documented why total sizes can differ even when the sync is perfect.
    • +
  • Enhancement: Added SERVER_NAME SERVER_ADDR SERVER_ADMIN variables and values to the output.
    • +
  • Enhancement: Added the local ip address for the imap connexions. It can help configuring firewalls to allow the imap source IP.
    • +
  • Enhancement: Added "Posta inviata" for \Sent with --automap
    • + + +
  • CGI context: Upped ERRORS_MAX_CGI from 20 to 500.
    • +
  • CGI context: Added --var to get values from proximapsync for variables REMOTE_ADDR REMOTE_HOST HTTP_REFERER HTTP_USER_AGENT SERVER_SOFTWARE SERVER_PORT HTTP_COOKIE
    • + +
  • Usability: Added warning "parsing headers of folder ... It can take time for huge folders. Be patient."
    • + +
  • Bug fix: Do not turn on --delete2duplicates when --syncduplicates is on, unless --delete2duplicates is given.
    • +
  • Bug fix: Redirect STDERR to STDOUT in all cases, --log or --nolog
    • +
  • Bug fix: Do not allow --skipcrossduplicates and --usecache. Exit EX_USAGE 64
    • +
  • Bug fix: Dates were not displayed under Windows because POSIX::strftime %e is not portable. Fixed several other places where dates were wrong on Windows.
    • +
  • Bug fix: --tests --testslive was listing 0 folders.
    • + + +
  • Refactoring: Deglobalized $debuglist $debugflags
    • +
  • Refactoring: Added sync->{ permanentflags2 }
    • + + +
+
  • 2.178
    • -
  • General:
  • Dependencies removed: None
  • Dependencies added: Perl module Compress::Zlib
    • @@ -107,10 +136,9 @@ If you are very interrested in those projects, don't hesitate to solicit me! --keepalive1 is on by default since imapsync release 2.169 Use --nokeepalive1 to disable it. Same thing for --keepalive2 but for --host2. See https://metacpan.org/pod/Mail::IMAPClient#Keepalive -
  • Enhancement:
    • -
  • Enhancement:
  • Usability: Removed --fast option. It didn't do anything anyway, it was a fake option.
    • +
  • Usability: Added advice inline advice on errors
    • ERR_OVERQUOTA. "The destination mailbox is 100% full, get free space on it and then resume the sync."
      • @@ -125,10 +153,6 @@ If you are very interrested in those projects, don't hesitate to solicit me!
    • Usability: Bytes sizes human readable are now presented in KB MB GB TB PB (1000 base) instead of KiB MiB GiB TiB PiB (1024 base)
    • Usability: Added README help for --truncmess
      • -
    • Usability:
      • -
    • Usability:
      • -
    • Usability:
      • -
    • Usability:
    • CGI context: Experimental loaddelay forced to 0 (no delay at all). So far a huge load is rarely a problem.
      • @@ -141,13 +165,10 @@ If you are very interrested in those projects, don't hesitate to solicit me!
    • Bug fix: Load average on Mac was sometimes buggy. locale float 3,14 instead of 3.14
    • Bug fix: errors_incr() exited with CATCH_ALL
    • Bug fix: Memory consumption on Mac was not relevant. Use RSS instead of VSZ, on Mac only. Linux stays with VSZ.
      • -
    • Bug fix:
      • -
    • Bug fix:
    • Docker context: Current working directory changed from /var/tmp to /var/tmp/uid_$EFFECTIVE_USER_ID
      • -
    @@ -1018,7 +1039,7 @@ by ignoring PERMANENTFLAGS (Exchange tests)
    • This document last modified on -($Id: news.shtml,v 1.83 2022/01/13 18:36:12 gilles Exp gilles $)
    +($Id: news.shtml,v 1.84 2022/04/04 16:11:25 gilles Exp gilles $)
    Top of the page

    diff --git a/TODO b/TODO index 8a865ef..0726a18 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,5 @@ #!/bin/cat -# $Id: TODO,v 1.262 2022/01/11 12:56:18 gilles Exp gilles $ +# $Id: TODO,v 1.265 2022/02/22 15:16:48 gilles Exp gilles $ This documentation is also at http://imapsync.lamiral.info/#doc @@ -9,9 +9,23 @@ TODO file for imapsync https://imapsync.lamiral.info/TODO + +SUGGESTED 2022_02_19 +Talk about size discrepencies. +Gmail doesn't count duplicates sizes twice. +Servers reported message sizes are always not the actual messages sizes. +Where put that information? + +SUGGESTED 2022_01_26 by Kuch/Mitchell/D. +Make --delete1 be optionally "Move to Trash folder" +Use MOVE if possible +https://datatracker.ietf.org/doc/html/rfc6851 + SUGGESTED 2022_01_09 by Neustradamus Add SCRAM-SHA* support https://github.com/imapsync/imapsync/issues/313 +https://perldoc.perl.org/Digest::SHA + SUGGESTED 2021_06_13 by rotemes https://github.com/imapsync/imapsync/issues/288 @@ -530,7 +544,7 @@ Write a Windows tutorial, TUTORIAL_Windows.html. Write a good practices migration tips document, GOOD_PRACTICES.html -Add uidexpunge with --delete if possible (like with --delete2). +Add uidexpunge with --delete1 if possible (like with --delete2). Add the possibility to specify a unity with --maxsize and --minsize Example --maxsize 10MB --minsize 10KB diff --git a/VERSION b/VERSION index fd1e1d9..dd8ca31 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.178 +2.200 diff --git a/X/cgi_memo b/X/cgi_memo index 0a4833e..67f0c16 100644 --- a/X/cgi_memo +++ b/X/cgi_memo @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: cgi_memo,v 1.74 2021/11/25 11:25:13 gilles Exp gilles $ +# $Id: cgi_memo,v 1.78 2022/03/28 09:24:08 gilles Exp gilles $ if test -n "$1"; then echoq() { echo "$@" ; } # not quiet mode @@ -154,10 +154,10 @@ longest_transfer() LC_ALL=C printf "%.0f\n" `datamash_file_op_index "$statsfile" max 4` } -echoq number_and_pids_of_imapsync_running -number_and_pids_of_imapsync_running() -{ - echo "`number_of_imapsync_running` : `pids_of_imapsync_running`" + +echoq pids_of_imapsync_running +pids_of_imapsync_running() { + pgrep -d ' ' -f cgi-bin/imapsync : # always return true } @@ -168,12 +168,25 @@ number_of_imapsync_running() : # always return true } -echoq pids_of_imapsync_running -pids_of_imapsync_running() { - pgrep -d ' ' -f cgi-bin/imapsync +echoq number_and_pids_of_imapsync_running +number_and_pids_of_imapsync_running() +{ + echo "`number_of_imapsync_running` : `pids_of_imapsync_running`" : # always return true } +echoq memory_used_by_all_imapsync_KiB +memory_used_by_all_imapsync_KiB() +{ + # Sum up all memory taken by imapsync runs, in KiB. + pids_of_imapsync_running=`pids_of_imapsync_running` + if test -n "$pids_of_imapsync_running" ; then + ps -o vsz -p $pids_of_imapsync_running | sed 1,1d | datamash sum 1 + else + echo 0 + fi +} + echoq oom_immune_imapsync_running oom_immune_imapsync_running() { for pid in `pids_of_imapsync_running` @@ -335,43 +348,41 @@ grep_all_stat_from_patterns_list() { sum_first_column_G_HTTP_USER_AGENT_sorted() { - awk '{sum += $1} END {print sum}' G_HTTP_USER_AGENT_sorted.txt + awk '{sum += $1} END {print sum}' G_HTTP_USER_AGENT_${1}_sorted.txt } stat_useragent_X() { - grep -o 'HTTP_USER_AGENT.*' G_HTTP_USER_AGENT.txt \ + grep -o 'HTTP_USER_AGENT.*' G_HTTP_USER_AGENT_$1.txt \ | tail -10000000 | sort | egrep -o -w 'Mozilla/5.0 \([^;]+' \ | sort | egrep -o '\([a-zA-Z]+' | sort | uniq -c | sort -g \ - | grep -v KHTML | tr -d '(' > G_HTTP_USER_AGENT_sorted.txt - - + | grep -v KHTML | tr -d '(' > G_HTTP_USER_AGENT_${1}_sorted.txt } echoq 'percent_stat_useragent_X' percent_stat_useragent_X() { - stat_useragent_X - sum_first_column_G_HTTP_USER_AGENT=`sum_first_column_G_HTTP_USER_AGENT_sorted` + stat_useragent_X "$1" + sum_first_column_G_HTTP_USER_AGENT=`sum_first_column_G_HTTP_USER_AGENT_sorted $1` { while read num_useragent useragent ; do #echo KK $num_useragent $useragent PerCent=`echo "scale=2; 100*$num_useragent/$sum_first_column_G_HTTP_USER_AGENT" | bc -l` echo "$useragent $PerCent % ( $num_useragent / $sum_first_column_G_HTTP_USER_AGENT )" done - } < G_HTTP_USER_AGENT_sorted.txt + } < G_HTTP_USER_AGENT_${1}_sorted.txt } stat_load() { - echo -n 'Load 1 min 5 min 15 min ' ; grep -o 'on.*cores' G_Load.txt|uniq - echo -n 'Load min: ' ; datamash --format=%3.1f -W min 3 min 4 min 5 < G_Load.txt - echo -n 'Load q1: ' ; datamash --format=%3.1f -W q1 3 q1 4 q1 5 < G_Load.txt - echo -n 'Load median: ' ; datamash --format=%3.1f -W median 3 median 4 median 5 < G_Load.txt - echo -n 'Load mean: ' ; datamash --format=%3.1f -W mean 3 mean 4 mean 5 < G_Load.txt - echo -n 'Load q3: ' ; datamash --format=%3.1f -W q3 3 q3 4 q3 5 < G_Load.txt - echo -n 'Load max: ' ; datamash --format=%3.1f -W max 3 max 4 max 5 < G_Load.txt + echo -n 'Load 1 min 5 min 15 min ' ; grep -o 'on.*cores' G_Load_$1.txt | uniq + echo -n 'Load min: ' ; datamash --format=%3.1f -W min 3 min 4 min 5 < G_Load_$1.txt + echo -n 'Load q1: ' ; datamash --format=%3.1f -W q1 3 q1 4 q1 5 < G_Load_$1.txt + echo -n 'Load median: ' ; datamash --format=%3.1f -W median 3 median 4 median 5 < G_Load_$1.txt + echo -n 'Load mean: ' ; datamash --format=%3.1f -W mean 3 mean 4 mean 5 < G_Load_$1.txt + echo -n 'Load q3: ' ; datamash --format=%3.1f -W q3 3 q3 4 q3 5 < G_Load_$1.txt + echo -n 'Load max: ' ; datamash --format=%3.1f -W max 3 max 4 max 5 < G_Load_$1.txt } echoq stat_exit_value @@ -432,49 +443,82 @@ stat_any() { echoq stat_all stat_all() { - stat_load ; echo + stat_load "$1" ; echo - # stat_any G_REMOTE_ADDR.txt - # stat_any G_REMOTE_HOST.txt - # stat_any G_HTTP_COOKIE.txt - # stat_any G_HTTP_REFERER.txt - - # See various_usefull() - # stat_any G_Host1_IMAP_server.txt - # stat_any G_Host2_IMAP_server.txt + echo G_REMOTE_ADDR_$1.txt + egrep -o 'REMOTE_ADDR is .*' G_REMOTE_ADDR_$1.txt | sort -g | uniq -c | sort -g | tail -5 - # stat_any G_Host1_banner.txt - # stat_any G_Host2_banner.txt + echo + echo G_REMOTE_HOST_$1.txt + egrep -o 'REMOTE_HOST is .*' G_REMOTE_HOST_$1.txt | sort -g | uniq -c | sort -g | tail -5 - stat_any G_Host1_Nb_messages.txt - stat_any G_Host2_Nb_messages.txt - stat_any G_Messages_transferred.txt - stat_any G_Messages_skipped.txt - stat_any G_Messages_found_in_host1_not_in_host2.txt 9 - stat_any G_Messages_found_in_host2_not_in_host1.txt 9 + echo + echo G_HTTP_COOKIE_$1.txt + egrep -o 'imapsync_runs=[0-9]+' G_HTTP_COOKIE_$1.txt | egrep -o '[0-9]+' | sort -n | tail -1 + + # stat_any G_HTTP_REFERER.txt + echo + echo G_HTTP_REFERER_$1.txt + egrep -o 'HTTP_REFERER is .*' G_HTTP_REFERER_$1.txt | sort -g | uniq -c | sort -g + + echo + echo G_Host1_IMAP_server_$1.txt + cat G_Host1_IMAP_server_$1.txt | datamash -s -W -g 4 count 4 | awk '{ print $2 " " $1 }' | sort -g | tail -5 + + echo + echo G_Host2_IMAP_server_$1.txt + cat G_Host2_IMAP_server_$1.txt | datamash -s -W -g 4 count 4 | awk '{ print $2 " " $1 }' | sort -g | tail -5 + + + echo + stat_any G_Host1_Nb_messages_$1.txt + stat_any G_Host2_Nb_messages_$1.txt + stat_any G_Messages_transferred_$1.txt + stat_any G_Messages_skipped_$1.txt + stat_any G_Messages_found_in_host1_not_in_host2_$1.txt 9 + stat_any G_Messages_found_in_host2_not_in_host1_$1.txt 9 # stat_any G_Folders_synced.txt - egrep -o '[0-9]+/[0-9]+ synced' G_Folders_synced.txt | egrep -o '^[0-9]+' > G_Folders_synced_.txt - egrep -o '[0-9]+/[0-9]+ synced' G_Folders_synced.txt | egrep -o '[0-9]+/[0-9]+' | egrep -o '[0-9]+$' > G_Folders_total_seen.txt - stat_any G_Folders_synced_.txt 1 - stat_any G_Folders_total_seen.txt 1 + egrep -o '[0-9]+/[0-9]+ synced' G_Folders_synced_$1.txt | egrep -o '^[0-9]+' > G_Folders_synced__$1.txt + egrep -o '[0-9]+/[0-9]+ synced' G_Folders_synced_$1.txt | egrep -o '[0-9]+/[0-9]+' | egrep -o '[0-9]+$' > G_Folders_total_seen_$1.txt + stat_any G_Folders_synced__$1.txt 1 + stat_any G_Folders_total_seen_$1.txt 1 # - stat_any G_Transfer_time.txt - stat_any G_Host1_Total_size.txt - stat_any G_Host2_Total_size.txt - stat_any G_Total_bytes_transferred.txt 5 - stat_any G_Message_rate.txt - stat_any G_Average_bandwidth_rate.txt 5 - stat_any G_Biggest_message.txt - stat_any G_Detected_errors.txt 2 + stat_any G_Transfer_time_$1.txt + stat_any G_Host1_Total_size_$1.txt + stat_any G_Host2_Total_size_$1.txt + stat_any G_Total_bytes_transferred_$1.txt 5 + stat_any G_Message_rate_$1.txt + stat_any G_Average_bandwidth_rate_$1.txt 5 + stat_any G_Biggest_message_$1.txt + stat_any G_Detected_errors_$1.txt 2 #stat_any G_Exiting_with_return_value.txt 5 # GROUP - stat_any G_Memory_consumption_at_the_end.txt 7 + stat_any G_Memory_consumption_at_the_end_$1.txt 7 #stat_any G_failure_Error_login.txt - percent_stat_useragent_X ; echo - stat_exit_value - echo "Data made at" `date -r grep_stats.txt` + echo cpu time + stat_any G_CPU_time_and_cpu_$1.txt 6 + echo '%allcpus' + stat_any G_CPU_time_and_cpu_$1.txt 10 + + echo G_Host1_banner_$1.txt + server_survey_percent G_Host1_banner_$1.txt | tail -6 + + echo + echo G_Host2_banner_$1.txt + server_survey_percent G_Host2_banner_$1.txt | tail -6 + + echo + echo USER_AGENT + percent_stat_useragent_X $1 ; + + echo + echo EXIT values + stat_exit_value $1 + + echo + echo "Data made at" `date -r grep_stats_$1.txt` } stat_transfer_time_mean() @@ -1217,20 +1261,49 @@ div_1_by_2_or_zero() fi } + +load_1m_linux() +{ + cat /proc/loadavg | cut -d' ' -f1 +} + +load_1m_freebsd() +{ + /sbin/sysctl vm.loadavg | egrep -o '[0-9]+\.[0-9]++' | head -1 +} + +load_1m() +{ + here_is_linux && load_1m_linux + here_is_freebsd && load_1m_freebsd +} + +KiBytes_to_Bytes() +{ + expr 1024 \* $1 +} + echoq number_of_imapsync_running_bandwidth number_of_imapsync_running_bandwidth() { + # Maybe I could do two number_of_imapsync_running one before + # one after and average the two. nir=`number_of_imapsync_running` + load_1m=`load_1m` nbsr=`number_of_bytes_sent_received_per_second_during ${1:-1}` + mubaiKiB=`memory_used_by_all_imapsync_KiB` + mubaiB=`KiBytes_to_Bytes $mubaiKiB` + mubai_human=`bytestohuman $mubaiB` ratio=`div_1_by_2_or_zero $nbsr $nir` date=`date_ymdhms` date_u=`LANG= date -u` nbsr_human=`bytestohuman $nbsr` ratio_human=`bytestohuman $ratio` - echo "$date $nir $nbsr $ratio $nbsr_human$ratio_human" - echo "Current number of syncs: $nir; Current total bandwidth: $nbsr_human/s; Current bandwidth per sync: $ratio_human/s; Current date/time: $date_u; ">/var/tmp/imapsync_current.txt + echo "$date $nir $nbsr $ratio $nbsr_human $ratio_human $load_1m $mubaiKiB" + echo "Load: $load_1m; Mem: $mubai_human; Number of syncs: $nir; Total bandwidth: $nbsr_human/s; Bandwidth per sync: $ratio_human/s; Date/time: $date_u; ">/var/tmp/imapsync_current.txt } + echoq loop_number_of_imapsync_running_bandwidth loop_number_of_imapsync_running_bandwidth() { @@ -1249,35 +1322,30 @@ echoq various_usefull various_usefull() { cat <<'EOF' strace -e trace=signal -f `pgrep /usr/sbin/apach | xargs -n1 echo -n " -p "` 2>&1 -egrep -o '[0-9]+/[0-9]+' G_Folders_synced.txt | sort -g -egrep -o '[0-9]+/[0-9]+' G_Folders_synced.txt | sort -t/ -g -k2 | uniq -c egrep -o '* ID .*' G_Read___ID.txt | sort | uniq -c | sort -n -egrep -o 'imapsync_runs=[0-9]+' G_HTTP_COOKIE.txt | egrep -o '[0-9]+' | sort -n | uniq -c | sort -g -k1,2 -egrep -o 'HTTP_REFERER is .*' G_HTTP_REFERER.txt | sort -g | uniq -c | sort -g -egrep -o 'REMOTE_HOST is .*' G_REMOTE_HOST.txt | sort -g | uniq -c | sort -g -egrep -o 'REMOTE_ADDR is .*' G_REMOTE_ADDR.txt | sort -g | uniq -c | sort -g -datamash -s -W -g 4 count 4 < G_Host1_IMAP_server.txt | awk '{ print $2 " " $1 }' | sort -g | tail -22 -datamash -s -W -g 4 count 4 < G_Host2_IMAP_server.txt | awk '{ print $2 " " $1 }' | sort -g | tail -22 egrep -o '* ID .*' G_Read___ID.txt | sort | awk '{ print $1 " " $2 " " $3 " NIL" }' | datamash -s -W -g 3 count 3 | awk '{ print $2 " " $1 }' | sort -g + locate perl.core | xargs -n 1 gdb -q -x /tmp/gdb_quit.txt -c + zcat /var/log/apache/httpd-access.log.*.gz|egrep -o -w 'Mozilla/5.0 \([^;]+' | sort | egrep -o '\([a-zA-Z]+' | sort | uniq -c | sort -g | grep -v KHTML zcat /var/log/apache/httpd-access.log.*.gz|grep 'POST /cgi-bin/imapsync' | egrep -o -w 'Mozilla/5.0 \([^;]+' | sort | egrep -o '\([a-zA-Z]+' | sort | uniq -c | sort -g | grep -v KHTML egrep -o '\[.+@[^]]+]' G_success_login.txt |head -222222 | sort | uniq -c | sort -g +cat G_success_login_on.txt | ./domains_servers_map | sort | uniq -c | sort -g list_all_logs |tail -9999 | xargs grep -i 'Exiting with return value 112' | tee Error_112_last_9999_syncs.txt cut -d: -f1 Error_112_last_30_days.txt | xargs grep -oih 'Invalid system flag.*' | sort | uniq -c list_all_logs | xargs grep -i 'Exiting with return value 112' | tee Error_112_all_syncs.txt cut -d: -f1 Error_112_all_syncs.txt | tail -100 | xargs egrep -oih 'Invalid system flag [^(]+' | sort | uniq -c -cat G_success_login_on.txt | ./domains_servers_map | sort | uniq -c | sort -g + logfiles_finished_recently -300| xargs grep -i 'Exiting with return value 10 ' | grep -v 'return value 0 ' | cut -d: -f1 | xargs tail -11 | grep 'failure: can not open imap connection on' | uniq -c | sort -g | grep http | tee ../http_host_failures.txt # Searching big messages copied over 500 MB list_all_logs|tail -50000 | xargs egrep '{.?[56789]........} copied' # online processes stats -cat /var/tmp/number_of_imapsync_running.txt | datamash -W min 2 max 2 mean 2 median 2 q1 2 q3 2 +cat /var/tmp/number_of_imapsync_running_every_60s.txt | datamash -W min 2 max 2 mean 2 median 2 q1 2 q3 2 for v in 2 3 4; do cat /var/tmp/number_of_imapsync_running_every_6s.txt | datamash --format=%10.0f -W min $v max $v mean $v median $v q1 $v q3 $v ; done netstat -I em0 -b -n -w 6 -q 1 while :; do ssh root@ks5 'cd /var/tmp/imapsync_cgi/ ; . cgi_memo ; loop_number_of_imapsync_running_bandwidth 6' ; echo $?; done @@ -1289,6 +1357,20 @@ cat G_Host1_Nb_messages.txt | sort -g -k4 | grep 202[01] |tail -100 | cut -f1 - # Best bandwidth moments cat /var/tmp/number_of_imapsync_running_every_60s.txt | sort -k3 -g| tail -66 + +# Sort by number of parallel runs and by load in case of equality +cat /var/tmp/number_of_imapsync_running_every_60s.txt | grep ^2022_ | sort -k2 -k9 -g | tail -666 + +# Sort by number of parallel runs and by bandwidth in case of equality +cat /var/tmp/number_of_imapsync_running_every_60s.txt | grep ^2022_ | sort -k2 -k3 -g | tail -666 + + +# getrusage on FreeBSD, espacially disk i/o +procstat -r `pgrep -f cgi-bin/imapsync` + +# Sum up all memory taken by imapsync runs, in KiB. +ps -o vsz -p `pgrep -f cgi-bin/imapsync` | sed 1,1d | datamash sum 1 +memory_used_by_all_imapsync_KiB EOF } diff --git a/X/imapsync_form.js b/X/imapsync_form.js index 5de5797..66c5b38 100644 --- a/X/imapsync_form.js +++ b/X/imapsync_form.js @@ -1,5 +1,5 @@ -// $Id: imapsync_form.js,v 1.26 2021/12/18 19:01:52 gilles Exp gilles $ +// $Id: imapsync_form.js,v 1.27 2022/01/30 20:57:46 gilles Exp gilles $ /*jslint browser: true*/ /*global $*/ @@ -751,9 +751,9 @@ $(document).ready( if ( "imapsync.lamiral.info" === location.hostname ) { - $( "#status_24h" ).attr('src', 'https://lstu.fr/imapsync_online_status_24h_1200x70') ; - $( "#status_7d" ).attr('src', 'https://lstu.fr/imapsync_online_status_7d') ; - $( "#status_2m" ).attr('src', 'https://lstu.fr/imapsync_online_status_2months_1200x70') ; + // $( "#status_24h" ).attr('src', 'https://lstu.fr/imapsync_online_status_24h_1200x70') ; + // $( "#status_7d" ).attr('src', 'https://lstu.fr/imapsync_online_status_7d') ; + // $( "#status_2m" ).attr('src', 'https://lstu.fr/imapsync_online_status_2months_1200x70') ; $( "#local_bandwidth" ).collapse( "show" ) ; $( "#local_status_dbmon" ).collapse( "show" ) ; $( "#local_status_hetrix" ).collapse( "show" ) ; diff --git a/X/imapsync_form_extra.html b/X/imapsync_form_extra.html index 2f04c51..f2e94e6 100644 --- a/X/imapsync_form_extra.html +++ b/X/imapsync_form_extra.html @@ -1,6 +1,6 @@ - + @@ -103,6 +103,9 @@ @@ -231,6 +234,9 @@ @@ -401,36 +407,9 @@

    Imapsync Online Status over the last 24h
    - Imapsync Online Status + Imapsync Online Status over the last 24h +

    - -

    - Imapsync Online Status over the last week
    - Imapsync Online Status -

    - -

    - Imapsync Online Status over the last two months
    - Imapsync Online Status
    -

    - - - - - @@ -463,7 +442,7 @@ monitor page powered by the Consoles --> Bottom
    - ($Id: imapsync_form_extra.html,v 1.22 2021/12/18 19:03:46 gilles Exp gilles $)
    + ($Id: imapsync_form_extra.html,v 1.25 2022/02/18 17:06:24 gilles Exp gilles $)
    Terms and conditions for anything: No limits to do anything with this work and this license!
    diff --git a/X/imapsync_form_extra_free.html b/X/imapsync_form_extra_free.html index e11be78..cc4581a 100644 --- a/X/imapsync_form_extra_free.html +++ b/X/imapsync_form_extra_free.html @@ -1,6 +1,6 @@ - + @@ -161,6 +161,9 @@ It's ok if you don't have a VAT number.
    @@ -289,6 +292,9 @@ It's ok if you don't have a VAT number.
    @@ -460,36 +466,9 @@ It's ok if you don't have a VAT number.

    Imapsync Online Status over the last 24h
    - Imapsync Online Status + Imapsync Online Status over the last 24h +

    - -

    - Imapsync Online Status over the last week
    - Imapsync Online Status -

    - -

    - Imapsync Online Status over the last two months
    - Imapsync Online Status
    -

    - - - - - @@ -522,7 +501,7 @@ monitor page powered by the Consoles --> Bottom
    - ($Id: imapsync_form_extra_free.html,v 1.23 2021/12/18 19:03:46 gilles Exp gilles $)
    + ($Id: imapsync_form_extra_free.html,v 1.27 2022/02/20 09:37:45 gilles Exp gilles $)
    Terms and conditions for anything: No limits to do anything with this work and this license!
    diff --git a/X/imapsync_online_status.html b/X/imapsync_online_status.html new file mode 100644 index 0000000..9076171 --- /dev/null +++ b/X/imapsync_online_status.html @@ -0,0 +1,50 @@ + + + + + + + + Imapsync Online Status + + + + + +

    Imapsync Online Status over the last 24h

    +

    + Imapsync Online Status over the last 24h +

    + +

    Imapsync Online Status over the last week

    +

    + Imapsync Online Status over the last week +

    + +

    Imapsync Online Status over the last two months

    +

    + Imapsync Online Status over the last two months
    +

    + +

    Imapsync Online Status over the last year

    +

    + Imapsync Online Status
    +

    + + + + + + + + + + \ No newline at end of file diff --git a/examples/imapsync_example.bat b/examples/imapsync_example.bat index 8e5b6b4..6d44041 100644 --- a/examples/imapsync_example.bat +++ b/examples/imapsync_example.bat @@ -1,48 +1,51 @@ -@REM $Id: imapsync_example.bat,v 1.11 2020/01/07 00:45:36 gilles Exp gilles $ +@REM $Id: imapsync_example.bat,v 1.12 2022/02/12 11:00:09 gilles Exp gilles $ -@REM imapsync example batch for Windows users -@REM lines beginning with @REM are just comments -@REM Please read them, they are written for you, human folk. +@REM Here is an imapsync example batch for Windows users. +@REM Lines beginning with @REM are just comments. +@REM Please read the comments, they are written for you, human folk. @REM Read also https://imapsync.lamiral.info/README_Windows.txt @REM for more details on how to use imapsync on Windows. -@REM Now let us enter the real work to fit your needs -@REM Replace below the 6 parameters +@REM Now let us enter the real work to fit your needs. +@REM Replace below the 6 parameters @REM "test1.lamiral.info" "test1" "secret1" "test2.lamiral.info" "test2" "secret2" -@REM with your own values -@REM Double quotes are necessary if a value contain one or more blanks. +@REM with your values. +@REM Double quotes are necessary if a value contains one or more blanks. -@REM value "test1.lamiral.info" for --host1 is the IMAP source server hostname or IP address -@REM value "test1" for --user1 is the IMAP source user login -@REM value "secret1" for --password1 is the IMAP source user password +@REM Value "test1.lamiral.info" for --host1 is the IMAP source server hostname or IP address. +@REM Value "test1" for --user1 is the IMAP source user login. +@REM Value "secret1" for --password1 is the IMAP source user password. -@REM value "test2.lamiral.info" for --host2 is the IMAP destination server hostname or IP address -@REM value "test2" for --user2 is the IMAP destination user login -@REM value "secret2" for --password2 is the IMAP destination user password +@REM Value "test2.lamiral.info" for --host2 is the IMAP destination server hostname or IP address. +@REM Value "test2" for --user2 is the IMAP destination user login. +@REM Value "secret2" for --password2 is the IMAP destination user password. @REM Character ^ at the end of the first line is essential and means @REM "this command continues on the next line". You can add other lines @REM but don't forget ^ character lasting each line, except the last one. +@REM Also, there must be no other character after each lasting character ^ +@REM not even a blank invisible character, or you'll end up with an +@REM error "Unknown command" with what you added. @REM That is not all, keep on reading! @REM Three other options are in this example because they are good to start with @REM @REM --dry makes imapsync doing nothing, just print what would be done without --dry. -@REM so if you leave --dry then imapsync will not sync your data. +@REM So, if you keep --dry then imapsync will not sync your data. @REM --justfolders does only folders creations, it ignores messages. @REM This option is good to verify the folder mapping is good for you @REM without starting to copy/pollute folders with messages. @REM -@REM --automap guesses folders mapping, it wors for folders like -@REM "Sent", "Junk", "Drafts", "All", "Archive", "Flagged". +@REM --automap guesses folders mapping, it works for folders like +@REM "Sent", "Junk", "Drafts", "All", "Archive", "Flagged". @REM @REM I suggest/impose to start with --automap --justfolders --dry. @REM If the folder mapping you see in the output is not good then add @REM some options --f1f2 "folder1=folder2" -@REM in oder to fix it. +@REM to fix the mapping. @REM Once you are happy with the folder names on the destination, @REM remove --dry and have a run to create folders on host2. diff --git a/i3 b/i3 deleted file mode 100755 index 7921010..0000000 --- a/i3 +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -# $Id: i3,v 1.20 2021/07/02 11:34:25 gilles Exp gilles $ - -BASE=`dirname $0` -perl -I${BASE}/W/Mail-IMAPClient-3.43/lib ${BASE}/imapsync "$@" - diff --git a/imapsync b/imapsync index d30f181..75b4563 100755 --- a/imapsync +++ b/imapsync @@ -1,6 +1,6 @@ #!/usr/bin/env perl -# $Id: imapsync,v 2.178 2022/01/12 21:28:37 gilles Exp gilles $ +# $Id: imapsync,v 2.200 2022/04/04 16:05:55 gilles Exp gilles $ # structure # pod documentation # use pragmas @@ -25,7 +25,7 @@ and without duplicates. =head1 VERSION -This documentation refers to Imapsync $Revision: 2.178 $ +This documentation refers to Imapsync $Revision: 2.200 $ =head1 USAGE @@ -44,42 +44,38 @@ This documentation refers to Imapsync $Revision: 2.178 $ We sometimes need to transfer mailboxes from one imap server to one another. -Imapsync command is a tool allowing incremental and -recursive imap transfers from one mailbox to another. -If you don't understand the previous sentence, it's normal, -it's pedantic computer-oriented jargon. +Imapsync command is a tool allowing incremental and recursive imap +transfers from one mailbox to another. If you don't understand the +previous sentence, it's normal, it's pedantic computer-oriented +jargon. -All folders are transferred, recursively, meaning -the whole folder hierarchy is taken, all messages in them, -and all message flags (\Seen \Answered \Flagged etc.) -are synced too. +All folders are transferred, recursively, meaning the whole folder +hierarchy is taken, all messages in them, and all message flags (\Seen +\Answered \Flagged etc.) are synced too. -Imapsync reduces the amount of data transferred by not transferring -a given message if it already resides on the destination side. -Messages that are on the destination side but not on the -source side stay as they are. See the --delete2 -option to have strict sync and delete them. +Imapsync reduces the amount of data transferred by not transferring a +given message if it already resides on the destination side. Messages +that are on the destination side but not on the source side stay as +they are. See the --delete2 option to have strict sync and delete +them. -How imapsync know a message is already on both sides? -Same specific headers and the transfer is done only once. -By default, the identification headers are -"Message-Id:" and "Received:" lines -but this choice can be changed with the --useheader option, -most often a duplicate problem is solved by using ---useheader "Message-Id" +How does imapsync know a message is already on both sides? Same +specific headers and the transfer is done only once. By default, the +identification headers are "Message-Id:" and "Received:" lines but +this choice can be changed with the --useheader option, most often a +duplicate problem is solved by using --useheader "Message-Id" -All flags are preserved, unread messages will stay unread, -read ones will stay read, deleted will stay deleted. -In the IMAP protocol, a deleted message is not really deleted, -it is marked \Deleted and can be undelete. Real destruction -comes with the EXPUNGE or UIDEXPUNGE IMAP commands. +All flags are preserved, unread messages will stay unread, read ones +will stay read, deleted will stay deleted. In the IMAP protocol, a +deleted message is not deleted, it is marked \Deleted and can be +undeleted. Real destruction comes with the EXPUNGE or UIDEXPUNGE IMAP +commands. -You can abort the transfer at any time and restart it later, -imapsync works well with bad connections and interruptions, -by design. On a terminal hit Ctr-c twice within two seconds -to abort the program. Hit Ctr-c just once makes -imapsync reconnect to both imap servers. +You can abort the transfer at any time and restart it later, imapsync +works well with bad connections and interruptions, by design. On a +terminal hit Ctr-c twice within two seconds to abort the program. Hit +Ctr-c just once makes imapsync reconnect to both imap servers. How do you know the sync is finished and well done? When imapsync ends by itself it mentions it with lines like those: @@ -88,31 +84,51 @@ When imapsync ends by itself it mentions it with lines like those: Removing pidfile /tmp/imapsync.pid Log file is LOG_imapsync/2020_11_17_15_59_22_761_test1_test2.txt ( to change it, use --logfile filepath ; or use --nolog to turn off logging ) -If you don't have those lines it means that either the sync process is still -running (or eventually hanging indefinitely) or that it ended without -a whisper, a strong kill -9 on Linux for example. +If you don't have those lines it means that either the sync process is +still running (or eventually hanging indefinitely) or that it ended +without a whisper, a strong kill -9 on Linux for example. If you have those final lines then it means the sync process is properly finished. It may have encountered problems though. -A good synchronization is mentioned by some lines above the last ones, especially -those three lines: +A good synchronization is mentioned by some lines above the last ones, +especially those three lines: The sync looks good, all 1745 identified messages in host1 are on host2. There is no unidentified message on host1. Detected 0 errors +Imapsync mentions the total sizes of both accounts at the beginning of +the sync and also at the end. Sometimes, even with a strict sync, +those total sizes differ, and sometimes they differ a lot. The +difference is not a good criterion to conclude the sync went wrong. -A classical scenario is synchronizing a mailbox B from another mailbox A -where you just want to keep a strict copy of A in B. Strict meaning +Why? That's because message sizes given by the imap servers are not +always accurate, they are not always the same as the actual message +sizes of the messages transferred by imapsync. Imapsync use the sizes +given by the imap servers to calculate the big total size. They can +differ. In the early days, Imapsync used the sizes of the messages as +one of the criteria to identify the messages, different sizes implied +different messages; but it was a mistake, the same message had +different sizes on both sides sometimes, depending on the imap +servers. + +Another explanation for a big total size difference is that Gmail +doesn't count the size of duplicate messages across folders twice, +while imapsync does. + + +A classical scenario is synchronizing a mailbox B from another mailbox +A where you just want to keep a strict copy of A in B. Strict meaning all messages in A will be in B but no more. -For this, option --delete2 can be used, it deletes messages in the host2 -folder B that are not in the host1 folder A. If you also need to destroy -host2 folders that are not in host1 then use --delete2folders. See also ---delete2foldersonly and --delete2foldersbutnot to set up exceptions -on folders to destroy. INBOX will never be destroyed, it's a mandatory -folder in IMAP so imapsync doesn't even try to remove it. +For a strict synchronization, use the option --delete2. The option +--delete2 deletes the messages in the host2 folder B that are not in +the host1 folder A. If you also need to destroy host2 folders that are +not in host1 then use --delete2folders. See also --delete2foldersonly +and --delete2foldersbutnot to set up exceptions on folders to +destroy. INBOX will never be destroyed, it's a mandatory folder in +IMAP so imapsync doesn't even try to remove it. A different scenario is to delete the messages from the source mailbox after a successful transfer, it can be a good feature when migrating @@ -122,20 +138,19 @@ messages that arrived after a sync or that failed to be transferred. In that case, use the --delete1 option. Option --delete1 implies also the option --expunge1 so all messages marked deleted on host1 will be -deleted. In IMAP protocol deleting a message does not delete it, -it marks it with the flag \Deleted, allowing an undelete. Expunging -a folder removes, definitively, all the messages marked as \Deleted -in this folder. +deleted. In IMAP protocol deleting a message does not delete it, it +marks it with the flag \Deleted, allowing an undelete. Expunging a +folder removes, definitively, all the messages marked as \Deleted in +this folder. You can also decide to remove empty folders once all of their messages have been transferred. Add --delete1emptyfolders to obtain this behavior. - -Imapsync is not adequate for maintaining two active imap accounts -in synchronization when the user plays independently on both sides. -Use offlineimap (written by John Goerzen) or mbsync (written by -Michael R. Elkins) for a 2 ways synchronization. +Imapsync is not adequate for maintaining two active imap accounts in +synchronization when the user plays independently on both sides. Use +offlineimap (written by John Goerzen) or mbsync (written by Michael +R. Elkins) for a 2 ways synchronization. =head1 OPTIONS @@ -143,18 +158,20 @@ Michael R. Elkins) for a 2 ways synchronization. usage: imapsync [options] The standard options are the six values forming the credentials. -Three values on each side are needed in order to login into the IMAP -servers. These six values are a hostname, a username, and a password, two times. +Three values on each side are needed to login into the IMAP +servers. These six values are a hostname, a username, and a password, +two times. -Conventions used in the following descriptions of the options: +Here are the conventions used in the following descriptions of the +options: - str means string - int means integer number - flo means float number - reg means regular expression - cmd means command + str means a string + int means an integer number + flo means a float number + reg means a regular expression + cmd means a command - --dry : Makes imapsync doing nothing for real; it just print what + --dry : Makes imapsync do nothing for real; it just prints what would be done without --dry. =head2 OPTIONS/credentials @@ -270,9 +287,9 @@ Same thing for user2 password. --nomixfolders : Do not merge folders when host1 is case-sensitive while host2 is not (like Exchange). Only the first - similar folder is synced (example: with folders - "Sent", "SENT" and "sent" - on host1 only "Sent" will be synced to host2). + similar folder is synced. Example: with folders + "Sent", "SENT" and "sent" on host1, only "Sent" + will be synced to host2. --skipemptyfolders : Empty host1 folders are not created on host2. @@ -710,40 +727,41 @@ https://imapsync.lamiral.info/FAQ.d/FAQ.Gmail.txt =head1 SECURITY -You can use --passfile1 instead of --password1 to mention the -password since it is safer. With --password1 option, on Linux, -any user on your host can see the password by using the 'ps auxwwww' -command. Using a variable (like IMAPSYNC_PASSWORD1) is also -dangerous because of the 'ps auxwwwwe' command. So, saving -the password in a well protected file (600 or rw-------) is -the best solution. +You can use --passfile1 instead of --password1 to mention the password +since it is safer. With --password1 option, on Linux, any user on your +host can see the password by using the 'ps auxwwww' command. Using a +variable (like IMAPSYNC_PASSWORD1) is also dangerous because of the +'ps auxwwwwe' command. So, saving the password in a well protected +file (600 or rw-------) is the best solution. Imapsync activates ssl or tls encryption by default, if possible. What detailed behavior is under this "if possible"? Imapsync activates ssl if the well known port imaps port (993) is open -on the imap servers. If the imaps port is closed then it open a -normal (clear) connection on port 143 but it looks for TLS support -in the CAPABILITY list of the servers. If TLS is supported -then imapsync goes to encryption with STARTTLS. +on the imap servers. If the imaps port is closed then it open a normal +(clear) connection on port 143 but it looks for TLS support in the +CAPABILITY list of the servers. If TLS is supported then imapsync goes +to encryption with STARTTLS. If the automatic ssl and the tls detections fail then imapsync will -not protect against sniffing activities on the network, especially -for passwords. +not protect against sniffing activities on the network, especially for +passwords. -If you want to force ssl or tls just use --ssl1 --ssl2 or --tls1 --tls2 +If you want to force ssl or tls just use --ssl1 --ssl2 or --tls1 +--tls2 -See also the document FAQ.Security.txt in the FAQ.d/ directory -or at https://imapsync.lamiral.info/FAQ.d/FAQ.Security.txt +See also the document FAQ.Security.txt in the FAQ.d/ directory or at +https://imapsync.lamiral.info/FAQ.d/FAQ.Security.txt =head1 EXIT STATUS -Imapsync will exit with a 0 status (return code) if everything went good. -Otherwise, it exits with a non-zero status. That's classical Unix behavior. -Here is the list of the exit code values (an integer between 0 and 255). -In Bourne Shells, this exit code value can be retrieved within the variable -value "$?" if you read it just after the imapsync call. +Imapsync will exit with a 0 status (return code) if everything went +good. Otherwise, it exits with a non-zero status. That's classical +Unix behavior. Here is the list of the exit code values (an integer +between 0 and 255). In Bourne Shells, this exit code value can be +retrieved within the variable value "$?" if you read it just after the +imapsync call. The names reflect their meaning: @@ -781,10 +799,10 @@ egrep '^Readonly my.*\$EX' imapsync | egrep -o 'EX.*' | sed 's_^_ _' =head1 LICENSE AND COPYRIGHT -Imapsync is free, open, public but not always gratis software -cover by the NOLIMIT Public License, now called NLPL. -See the LICENSE file included in the distribution or just read this -simple sentence as it IS the licence text: +Imapsync is free, open, public but not always gratis software cover by +the NOLIMIT Public License, now called NLPL. See the LICENSE file +included in the distribution or just read the following simple +sentence as it IS the license text: "No limits to do anything with this work and this license." @@ -801,12 +819,11 @@ Gilles LAMIRAL Good feedback is always welcome. Bad feedback is very often welcome. -Gilles LAMIRAL earns his living by writing, installing, -configuring and sometimes teaching free, open and often gratis -software. Imapsync used to be "always gratis" but now it is -only "often gratis" because imapsync is sold by its author, -your servitor, a good way to maintain and support free open public -software tools over decades. +Gilles LAMIRAL earns his living by writing, installing, configuring +and sometimes teaching free, open, and often gratis software. Imapsync +used to be "always gratis" but now it is only "often gratis" because +imapsync is sold by its author, your servitor, a good way to maintain +and support free open public software tools over decades. =head1 BUGS AND LIMITATIONS @@ -818,10 +835,9 @@ See https://imapsync.lamiral.info/S/imapservers.shtml =head1 HUGE MIGRATION -If you have many mailboxes to migrate think about a little -shell program. Write a file called file.txt (for example) -containing users and passwords. -The separator used in this example is ';' +If you have many mailboxes to migrate think about a little shell +program. Write a file called file.txt (for example) containing users +and passwords. The separator used in this example is ';' The file.txt file contains: @@ -857,19 +873,19 @@ https://imapsync.lamiral.info/examples/ Imapsync works under most Windows (2000, XP, Vista, Seven, Eight, Ten and all Server releases 2000, 2003, 2008 and R2, 2012 and R2, 2016) - as a standalone binary software called imapsync.exe, - usually launched from a batch file in order to avoid always typing - the options. There is also a 32bit binary called imapsync_32bit.exe + as a standalone binary software called imapsync.exe, usually launched + from a batch file in order to avoid always typing the options. There + is also a 32bit binary called imapsync_32bit.exe - Imapsync works under OS X as a standalone binary - software called imapsync_bin_Darwin + Imapsync works under OS X as a standalone binary software called + imapsync_bin_Darwin Purchase latest imapsync at https://imapsync.lamiral.info/ - You'll receive a link to a compressed tarball called imapsync-x.xx.tgz - where x.xx is the version number. Untar the tarball where - you want (on Unix): + You'll receive a link to a compressed tarball called + imapsync-x.xx.tgz where x.xx is the version number. + Untar the tarball where you want (on Unix): tar xzvf imapsync-x.xx.tgz @@ -882,9 +898,8 @@ https://imapsync.lamiral.info/examples/ =head1 CONFIGURATION -There is no specific configuration file for imapsync, -everything is specified by the command line parameters -and the default behavior. +There is no specific configuration file for imapsync, everything is +specified by the command line parameters and the default behavior. =head1 HACKING @@ -932,24 +947,23 @@ List verified on Friday July 1, 2021. =head1 HISTORY -I initially wrote imapsync in July 2001 because an enterprise, -called BaSystemes, paid me to install a new imap server -without losing huge old mailboxes located in a far -away remote imap server, accessible by an -often broken low-bandwidth ISDN link. +I initially wrote imapsync in July 2001 because an enterprise, called +BaSystemes, paid me to install a new imap server without losing huge +old mailboxes located in a far away remote imap server, accessible by +an often broken low-bandwidth ISDN link. -I had to verify every mailbox was well transferred, all folders, all messages, -without wasting bandwidth or creating duplicates upon resyncs. The imapsync -design was made with the beautiful rsync command in mind. +I had to verify every mailbox was well transferred, all folders, all +messages, without wasting bandwidth or creating duplicates upon +resyncs. The imapsync design was made with the beautiful rsync command +in mind. -Imapsync started its life as a patch of the copy_folder.pl -script. The script copy_folder.pl comes from the Mail-IMAPClient-2.1.3 perl -module tarball source (more precisely in the examples/ directory of the +Imapsync started its life as a patch of the copy_folder.pl script. The +script copy_folder.pl comes from the Mail-IMAPClient-2.1.3 perl module +tarball source (more precisely in the examples/ directory of the Mail-IMAPClient tarball). -So many changes happened since then that I wonder -if it remains any lines of the original -copy_folder.pl in imapsync source code. +So many changes happened since then that I wonder if it remains any +lines of the original copy_folder.pl in imapsync source code. =cut @@ -1212,10 +1226,8 @@ sub comment_err_flags Readonly my $DEFAULT_LOGDIR => 'LOG_imapsync' ; -Readonly my $ERRORS_MAX => 50 ; # exit after 50 errors. -Readonly my $ERRORS_MAX_CGI => 20 ; # exit after 20 errors in CGI context. - - +Readonly my $ERRORS_MAX => 50 ; # exit after 50 errors. +Readonly my $ERRORS_MAX_CGI => 500 ; # exit after 500 errors in CGI context. Readonly my $INTERVAL_TO_EXIT => 2 ; # interval max to exit instead of reconnect @@ -1292,10 +1304,9 @@ Readonly my $FORCE => 1 ; # Currently working to finish with only $sync, $acc1, $acc2 # Not finished yet... -my( +my( $sync, $acc1, $acc2, - $debugflags, - $debuglist, $debugdev, $debugmaxlinelength, $debugcgi, + $debugdev, $debugmaxlinelength, $debugcgi, @include, @exclude, @folderrec, @folderfirst, @folderlast, @h1_folders_all, %h1_folders_all, @@ -1329,7 +1340,7 @@ my( $split1, $split2, $modulesversion, $delete2folders, $delete2foldersonly, $delete2foldersbutnot, - $usecache, $debugcache, $cacheaftercopy, + $debugcache, $cacheaftercopy, $wholeheaderifneeded, %h1_msgs_copy_by_uid, $useuid, $h2_uidguess, $checkmessageexists, $messageidnodomain, @@ -1338,7 +1349,6 @@ my( $minmaxlinelength, $fixcolonbug, $create_folder_old, - $skipcrossduplicates, $debugcrossduplicates, $disarmreadreceipts, $mixfolders, $fetch_hash_set, @@ -1376,7 +1386,7 @@ $acc2->{ N } = '2' ; $sync->{timestart} = time ; # Is a float because of use Time::HiRres -$sync->{rcs} = q{$Id: imapsync,v 2.178 2022/01/12 21:28:37 gilles Exp gilles $} ; +$sync->{rcs} = q{$Id: imapsync,v 2.200 2022/04/04 16:05:55 gilles Exp gilles $} ; $sync->{ memory_consumption_at_start } = memory_consumption( ) || 0 ; @@ -1451,10 +1461,9 @@ cgibuildheader( $sync ) ; docker_context( $sync ) ; print_output_if_needed( $sync ) ; - - output_reset_with( $sync ) ; + # don't go on if options are not all known. if ( ! defined $options_good ) { exit $EX_USAGE ; } @@ -1463,8 +1472,8 @@ if ( ! defined $options_good ) { exit $EX_USAGE ; } # the second line (ending with "1 ;") can then stay active or be commented, # the result will be the same: no releasecheck by default (because 0 is then the defined value). -$sync->{releasecheck} = defined $sync->{releasecheck} ? $sync->{releasecheck} : 0 ; -#$sync->{releasecheck} = defined $sync->{releasecheck} ? $sync->{releasecheck} : 1 ; +#$sync->{releasecheck} = defined $sync->{releasecheck} ? $sync->{releasecheck} : 0 ; +$sync->{releasecheck} = defined $sync->{releasecheck} ? $sync->{releasecheck} : 1 ; # just the version if ( $sync->{ version } ) { @@ -1482,6 +1491,9 @@ after_get_options( $sync, $options_good ) ; #local $ENV{TZ} = 'GMT' if ( under_cgi_context( $sync ) and 'MSWin32' ne $OSNAME ) ; #output( $sync, localtime(time) . " " . gmtime(time) . "\n" ) ; +make_var_array_to_a_hash( $sync ) ; + + # Under CGI environment, fix caveat emptor potential issues cgisetcontext( $sync ) ; @@ -1490,6 +1502,7 @@ get_options_extra( $sync ) ; # --gmail --gmail --exchange --office etc. easyany( $sync ) ; + $sync->{ sanitize } = defined $sync->{ sanitize } ? $sync->{ sanitize } : 1 ; sanitize( $sync ) ; @@ -1525,20 +1538,34 @@ $sync->{ log } = defined $sync->{ log } ? $sync->{ log } : $sync->{ errorsdump } = defined $sync->{ errorsdump } ? $sync->{ errorsdump } : 1 ; $sync->{ errorsmax } = defined $sync->{ errorsmax } ? $sync->{ errorsmax } : $ERRORS_MAX ; +$sync->{ emailreport1 } = defined $sync->{ emailreport1 } ? $sync->{ emailreport1 } : 1 ; +$sync->{ emailreport2 } = defined $sync->{ emailreport2 } ? $sync->{ emailreport2 } : 1 ; + + # log and output binmode STDOUT, ":encoding(UTF-8)" ; -if ( $sync->{ log } ) { - setlogfile( $sync ) ; - teelaunch( $sync ) ; +if ( $sync->{ log } ) { + $sync->{ logdir } = setlogdir( $sync ) ; + $sync->{ logfile } = setlogfile( $sync, $sync->{ logfile } ) ; + $sync->{ tee } = teelaunch( $sync, $sync->{ logfile } ) ; # now $sync->{tee} is a filehandle to STDOUT and the logfile -} +} #binmode STDERR, ":encoding(UTF-8)" ; -# STDERR goes to the same place: LOG and STDOUT (if logging is on) -# Useful only for --debugssl -$sync->{tee} and local *STDERR = *${$sync->{tee}}{IO} ; +# STDERR goes to the same place: LOG and STDOUT if logging is on +# or just STDOUT +# + +stderr_to_stdout( $sync ) ; + + +if ( usecache_and_skipcrossduplicates( $sync ) ) +{ + $sync->{ nb_errors }++ ; + exit_clean( $sync, $EX_USAGE, "Error: can not have both --usecache and --skipcrossduplicates\n" ) ; +} @@ -1568,11 +1595,8 @@ $sync->{ warn_release } = ( $sync->{ releasecheck } ) ? check_last_release( ) : $wholeheaderifneeded = defined $wholeheaderifneeded ? $wholeheaderifneeded : 1; # Activate --usecache if --useuid is set and there is no --nousecache -$usecache = 1 if ( $useuid and ( ! defined $usecache ) ) ; -$cacheaftercopy = 1 if ( $usecache and ( ! defined $cacheaftercopy ) ) ; - - - +$sync->{ usecache } = 1 if ( $useuid and ( ! defined $sync->{ usecache } ) ) ; +$cacheaftercopy = 1 if ( $sync->{ usecache } and ( ! defined $cacheaftercopy ) ) ; $sync->{ checkfoldersexist } = defined $sync->{ checkfoldersexist } ? $sync->{ checkfoldersexist } : 1 ; $checkmessageexists = defined $checkmessageexists ? $checkmessageexists : 0 ; @@ -1593,7 +1617,8 @@ $create_folder_old = defined $create_folder_old ? $create_folder_old : 0 ; $mixfolders = defined $mixfolders ? $mixfolders : 1 ; $sync->{automap} = defined $sync->{automap} ? $sync->{automap} : 0 ; -$sync->{ delete2duplicates } = 1 if ( $sync->{ delete2 } and ( ! defined $sync->{ delete2duplicates } ) ) ; +$sync->{ delete2duplicates } = determine_delete2duplicates( $sync ) ; + $sync->{maxmessagespersecond} = defined $sync->{maxmessagespersecond} ? $sync->{maxmessagespersecond} : 0 ; $sync->{maxbytespersecond} = defined $sync->{maxbytespersecond} ? $sync->{maxbytespersecond} : 0 ; @@ -1651,7 +1676,7 @@ cgiload( $sync ) ; $fixcolonbug = defined $fixcolonbug ? $fixcolonbug : 1 ; -if ( $usecache and $fixcolonbug ) { tmpdir_fix_colon_bug( $sync ) } ; +if ( $sync->{ usecache } and $fixcolonbug ) { tmpdir_fix_colon_bug( $sync ) } ; $modulesversion and myprint( "Modules version list ( use --no-modulesversion to turn off printing this Perl modules list ):\n", modulesversion(), "\n" ) ; @@ -1731,8 +1756,8 @@ my %SSL_VERIFY_STR ; Readonly $SSL_VERIFY_POLICY => IO::Socket::SSL::SSL_VERIFY_NONE( ) ; Readonly %SSL_VERIFY_STR => ( - IO::Socket::SSL::SSL_VERIFY_NONE( ) => 'SSL_VERIFY_NONE, ie, do not check the certificate server.' , - IO::Socket::SSL::SSL_VERIFY_PEER( ) => 'SSL_VERIFY_PEER, ie, check the certificate server' , + IO::Socket::SSL::SSL_VERIFY_NONE( ) => 'SSL_VERIFY_NONE, ie, do not check the server certificate.' , + IO::Socket::SSL::SSL_VERIFY_PEER( ) => 'SSL_VERIFY_PEER, ie, check the server certificate.' , ) ; $IO::Socket::SSL::DEBUG = defined( $sync->{debugssl} ) ? $sync->{debugssl} : 1 ; @@ -2202,7 +2227,7 @@ checkselectable( $sync ) ; -# Bugfix OpenFind folders named like "kk \*123" are in fact "kk *123" (no \) +# Local bugfix. OpenFind folders named like "kk \*123" are in fact "kk *123" (no \) #foreach my $folder ( @{ $sync->{ h1_folders_wanted } } ) #{ # $folder =~ s{ \\\*}{ *}g ; @@ -2211,7 +2236,8 @@ checkselectable( $sync ) ; # this hack is because LWP post does not pass well a hash in the $form parameter # but it does pass well an array -%{ $sync->{f1f2h} } = split_around_equal( @{ $sync->{f1f2} } ) ; +#%{ $sync->{f1f2h} } = split_around_equal( @{ $sync->{f1f2} } ) ; +make_f1f2_array_to_a_hash( $sync ) ; automap( $sync ) ; @@ -2448,8 +2474,8 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) my $h2_fold_nb_messages = count_from_select( @select_results ) ; myprint( "Host2: folder [$h2_fold] has $h2_fold_nb_messages messages in total (mentioned by SELECT)\n" ) ; - my $permanentflags2 = permanentflags( $sync, @select_results ) ; - myprint( "Host2: folder [$h2_fold] permanentflags: $permanentflags2\n" ) ; + $sync->{ permanentflags2 } = permanentflags( $sync, @select_results ) ; + myprint( "Host2: folder [$h2_fold] permanentflags: $sync->{ permanentflags2 }\n" ) ; if ( $sync->{ expunge1 } ) { @@ -2484,7 +2510,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) if ( ! reconnect_12_if_needed( $sync ) ) { last FOLDER ; } myprint( "Host1: folder [$h1_fold] considering $h1_msgs_nb messages\n" ) ; - ( $sync->{ debug } or $debuglist ) and myprint( "Host1: folder [$h1_fold] considering $h1_msgs_nb messages, LIST gives: @h1_msgs\n" ) ; + ( $sync->{ debug } or $sync->{ debuglist } ) and myprint( "Host1: folder [$h1_fold] considering $h1_msgs_nb messages, LIST gives: @h1_msgs\n" ) ; $sync->{ debug } and myprint( "Host1: selecting messages of folder [$h1_fold] took ", timenext( $sync ), " s\n" ) ; my $h2_msgs_all_hash_ref = { } ; @@ -2495,7 +2521,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) my $h2_msgs_nb = scalar @h2_msgs ; myprint( "Host2: folder [$h2_fold] considering $h2_msgs_nb messages\n" ) ; - ( $sync->{ debug } or $debuglist ) and myprint( "Host2: folder [$h2_fold] considering $h2_msgs_nb messages, LIST gives: @h2_msgs\n" ) ; + ( $sync->{ debug } or $sync->{ debuglist } ) and myprint( "Host2: folder [$h2_fold] considering $h2_msgs_nb messages, LIST gives: @h2_msgs\n" ) ; $sync->{ debug } and myprint( "Host2: selecting messages of folder [$h2_fold] took ", timenext( $sync ), " s\n" ) ; my $cache_base = "$sync->{ tmpdir }/imapsync_cache/" ; @@ -2508,7 +2534,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) if ( ! reconnect_12_if_needed( $sync ) ) { last FOLDER ; } - if ( $usecache ) { + if ( $sync->{ usecache } ) { myprint( "Local cache directory: $cache_dir ( " . length( $cache_dir ) . " characters long )\n" ) ; mkpath( "$cache_dir" ) ; ( $cache_1_2_ref, $cache_2_1_ref ) @@ -2544,35 +2570,38 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) if ( $useuid ) { # use uid so we have to avoid getting header @h1_msgs_copy_by_uid{ @h1_msgs_not_in_cache } = ( ) ; - @h2_msgs_delete2_not_in_cache = @h2_msgs_not_in_cache if $usecache ; + @h2_msgs_delete2_not_in_cache = @h2_msgs_not_in_cache if $sync->{ usecache } ; @h1_msgs_not_in_cache = ( ) ; @h2_msgs_not_in_cache = ( ) ; #myprint( "delete2: @h2_msgs_delete2_not_in_cache\n" ) ; } - - $sync->{ debug } and myprint( "Host1: parsing headers of folder [$h1_fold]\n" ) ; - - my ($h1_heads_ref, $h1_fir_ref) = ({}, {}); - $h1_heads_ref = $sync->{imap1}->parse_headers([@h1_msgs_not_in_cache], @useheader) if (@h1_msgs_not_in_cache); + + if ( $sync->{ debug } or ( 5000 <= scalar( @h1_msgs_not_in_cache ) ) ) + { + myprint( "Host1: parsing headers of folder [$h1_fold]. It can take time for huge folders. Be patient.\n" ) ; + } + + my ( $h1_heads_ref, $h1_fir_ref ) = ( {}, {} ) ; + $h1_heads_ref = $sync->{ imap1 }->parse_headers( [ @h1_msgs_not_in_cache ], @useheader ) if ( @h1_msgs_not_in_cache ) ; $sync->{ debug } and myprint( "Host1: parsing headers of folder [$h1_fold] took ", timenext( $sync ), " s\n" ) ; - @{ $h1_fir_ref }{@h1_msgs} = ( undef ) ; + @{ $h1_fir_ref }{ @h1_msgs } = ( undef ) ; - $sync->{ debug } and myprint( "Host1: getting flags idate and sizes of folder [$h1_fold]\n" ) ; + $sync->{ debug } and myprint( "Host1: getting flags idate and sizes of folder [$h1_fold]\n" ) ; my @h1_common_fetch_param = ( 'FLAGS', 'INTERNALDATE', 'RFC822.SIZE' ) ; if ( $sync->{ synclabels } or $sync->{ resynclabels } ) { push @h1_common_fetch_param, 'X-GM-LABELS' ; } - if ( $sync->{abletosearch1} ) + if ( $sync->{ abletosearch1 } ) { - $h1_fir_ref = $sync->{imap1}->fetch_hash( \@h1_msgs, @h1_common_fetch_param, $h1_fir_ref ) + $h1_fir_ref = $sync->{ imap1 }->fetch_hash( \@h1_msgs, @h1_common_fetch_param, $h1_fir_ref ) if ( @h1_msgs ) ; } else { my $fetch_hash_uids = $fetch_hash_set || "1:*" ; - $h1_fir_ref = $sync->{imap1}->fetch_hash( $fetch_hash_uids, @h1_common_fetch_param, $h1_fir_ref ) + $h1_fir_ref = $sync->{ imap1 }->fetch_hash( $fetch_hash_uids, @h1_common_fetch_param, $h1_fir_ref ) if ( @h1_msgs ) ; } @@ -2588,7 +2617,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) my @h1_msgs_duplicate; foreach my $m ( @h1_msgs_not_in_cache ) { - my $rc = parse_header_msg( $sync, $sync->{imap1}, $m, $h1_heads_ref, $h1_fir_ref, 'Host1', \%h1_hash ) ; + my $rc = parse_header_msg( $sync, $sync->{imap1}, $m, $h1_fold, $h1_heads_ref, $h1_fir_ref, 'Host1', \%h1_hash ) ; if ( ! defined $rc ) { my $h1_size = $h1_fir_ref->{$m}->{'RFC822.SIZE'} || 0; @@ -2619,30 +2648,31 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) $sync->{ debug } and myprint( 'Host1: whole time parsing headers took ', timenext( $sync ), " s\n" ) ; - - # Getting headers and metada can be so long that host2 might be disconnected here if ( ! reconnect_12_if_needed( $sync ) ) { last FOLDER ; } - $sync->{ debug } and myprint( "Host2: parsing headers of folder [$h2_fold]\n" ) ; + if ( $sync->{ debug } or ( 5000 <= scalar( @h2_msgs_not_in_cache ) ) ) + { + myprint( "Host2: parsing headers of folder [$h2_fold]. It can take time for huge folders. Be patient.\n" ) ; + } - my ($h2_heads_ref, $h2_fir_ref) = ( {}, {} ); - $h2_heads_ref = $sync->{imap2}->parse_headers([@h2_msgs_not_in_cache], @useheader) if (@h2_msgs_not_in_cache); + my ( $h2_heads_ref, $h2_fir_ref ) = ( {}, {} ); + $h2_heads_ref = $sync->{ imap2 }->parse_headers( [ @h2_msgs_not_in_cache ], @useheader ) if ( @h2_msgs_not_in_cache ); $sync->{ debug } and myprint( "Host2: parsing headers of folder [$h2_fold] took ", timenext( $sync ), " s\n" ) ; $sync->{ debug } and myprint( "Host2: getting flags idate and sizes of folder [$h2_fold]\n" ) ; - @{ $h2_fir_ref }{@h2_msgs} = ( ); # fetch_hash can select by uid with last arg as ref + @{ $h2_fir_ref }{ @h2_msgs } = ( ); # fetch_hash can select by uid with last arg as ref my @h2_common_fetch_param = ( 'FLAGS', 'INTERNALDATE', 'RFC822.SIZE' ) ; if ( $sync->{ synclabels } or $sync->{ resynclabels } ) { push @h2_common_fetch_param, 'X-GM-LABELS' ; } - if ( $sync->{abletosearch2} and scalar( @h2_msgs ) ) { - $h2_fir_ref = $sync->{imap2}->fetch_hash( \@h2_msgs, @h2_common_fetch_param, $h2_fir_ref) ; + if ( $sync->{ abletosearch2 } and scalar( @h2_msgs ) ) { + $h2_fir_ref = $sync->{ imap2 }->fetch_hash( \@h2_msgs, @h2_common_fetch_param, $h2_fir_ref ) ; }else{ my $fetch_hash_uids = $fetch_hash_set || "1:*" ; - $h2_fir_ref = $sync->{imap2}->fetch_hash( $fetch_hash_uids, @h2_common_fetch_param, $h2_fir_ref ) + $h2_fir_ref = $sync->{ imap2 }->fetch_hash( $fetch_hash_uids, @h2_common_fetch_param, $h2_fir_ref ) if ( @h2_msgs ) ; } @@ -2650,7 +2680,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) my @h2_msgs_duplicate; foreach my $m (@h2_msgs_not_in_cache) { - my $rc = parse_header_msg( $sync, $sync->{imap2}, $m, $h2_heads_ref, $h2_fir_ref, 'Host2', \%h2_hash ) ; + my $rc = parse_header_msg( $sync, $sync->{imap2}, $m, $h2_fold, $h2_heads_ref, $h2_fir_ref, 'Host2', \%h2_hash ) ; my $h2_size = $h2_fir_ref->{$m}->{'RFC822.SIZE'} || 0 ; if (! defined $rc ) { myprint( "Host2: $h2_fold/$m size $h2_size ignored (no wanted headers so we ignore this message)\n" ) ; @@ -2832,10 +2862,10 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) if ( ( not exists $h2_hash{ $m_id } ) and ( not ( exists $sync->{ h2_folders_of_md5 }->{ $m_id } ) - or not $skipcrossduplicates ) ) + or not $sync->{ skipcrossduplicates } ) ) { # copy - my $h2_msg = copy_message( $sync, $h1_msg, $h1_fold, $h2_fold, $h1_fir_ref, $permanentflags2, $cache_dir ) ; + my $h2_msg = copy_message( $sync, $h1_msg, $h1_fold, $h2_fold, $h1_fir_ref, $cache_dir ) ; if ( $h2_msg and $sync->{ delete1 } and not $sync->{ expungeaftereach } ) { # not expunged push @h1_msgs_to_delete, $h1_msg ; @@ -2868,7 +2898,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) { my $h2_msg = $h2_hash{$m_id}{'m'} ; $sync->{ debug } and myprint( "Host1: found that msg $h1_fold/$h1_msg equals Host2 $h2_fold/$h2_msg\n" ) ; - if ( $usecache ) + if ( $sync->{ usecache } ) { $debugcache and myprint( "touch $cache_dir/${h1_msg}_$h2_msg\n" ) ; touch( "$cache_dir/${h1_msg}_$h2_msg" ) @@ -2878,7 +2908,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) elsif( exists $sync->{ h2_folders_of_md5 }->{ $m_id } ) { my @folders_dup = keys %{ $sync->{ h2_folders_of_md5 }->{ $m_id } } ; - ( $sync->{ debug } or $debugcrossduplicates ) and myprint( "Host1: found that msg $h1_fold/$h1_msg is also in Host2 folders @folders_dup\n" ) ; + ( $sync->{ debug } or $sync->{ debugcrossduplicates } ) and myprint( "Host1: found that msg $h1_fold/$h1_msg is also in Host2 folders @folders_dup\n" ) ; $sync->{ h2_nb_msg_crossdup } +=1 ; } $sync->{ total_bytes_skipped } += $h1_size ; @@ -2890,7 +2920,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) #$debug and myprint( "MESSAGE $m_id\n" ) ; my $h2_msg = $h2_hash{$m_id}{'m'}; if ( $sync->{resyncflags} ) { - sync_flags_fir( $sync, $h1_fold, $h1_msg, $h2_fold, $h2_msg, $permanentflags2, $h1_fir_ref, $h2_fir_ref ) ; + sync_flags_fir( $sync, $h1_fold, $h1_msg, $h2_fold, $h2_msg, $h1_fir_ref, $h2_fir_ref ) ; } # Good my $h2_size = $h2_hash{$m_id}{'s'}; @@ -2925,7 +2955,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) $debugcache and myprint( "cache messages update flags $h1_msg->$h2_msg\n" ) ; if ( $sync->{resyncflags} ) { - sync_flags_fir( $sync, $h1_fold, $h1_msg, $h2_fold, $h2_msg, $permanentflags2, $h1_fir_ref, $h2_fir_ref ) ; + sync_flags_fir( $sync, $h1_fold, $h1_msg, $h2_fold, $h2_msg, $h1_fir_ref, $h2_fir_ref ) ; } my $h1_size = $h1_fir_ref->{ $h1_msg }->{ 'RFC822.SIZE' } || 0 ; $sync->{ total_bytes_skipped } += $h1_size; @@ -2945,7 +2975,7 @@ FOLDER: foreach my $h1_fold ( @{ $sync->{h1_folders_wanted} } ) $sync->{ debug } and myprint( "Copy by uid $h1_fold/$h1_msg\n" ) ; if ( ! reconnect_12_if_needed( $sync ) ) { last FOLDER ; } - my $h2_msg = copy_message( $sync, $h1_msg, $h1_fold, $h2_fold, $h1_fir_ref, $permanentflags2, $cache_dir ) ; + my $h2_msg = copy_message( $sync, $h1_msg, $h1_fold, $h2_fold, $h1_fir_ref, $cache_dir ) ; if( $sync->{ delete2 } and exists $h2_folders_from_1_several{ $h2_fold } and $h2_msg ) { myprint( "Host2: msg $h2_fold/$h2_msg will cancel deletion [fresh copy] on host2\n" ) ; $uid_candidate_no_deletion{ $h2_fold }{ $h2_msg }++ ; @@ -2985,9 +3015,6 @@ END_SIZE foldersizesatend( $sync ) ; } -#$sync->{imap1}->State( 0 ); # Unconnected -if ( ! lost_connection( $sync, $sync->{imap1}, "for host1 [$sync->{host1}]" ) ) { $sync->{imap1}->logout( ) ; } -if ( ! lost_connection( $sync, $sync->{imap2}, "for host2 [$sync->{host2}]" ) ) { $sync->{imap2}->logout( ) ; } do_and_print_stats( $sync ) ; @@ -3003,6 +3030,12 @@ if ( $sync->{ testslive } or $sync->{ testslive6 } ) tests_live_result( $sync->{ nb_errors } ) ; } +$sync->{ emailreport2 } and email_report_append( $sync->{ acc2 }, email_report( $sync, email_report_body_extra2( ) ) ) ; +$sync->{ emailreport1 } and email_report_append( $sync->{ acc1 }, email_report( $sync, email_report_body_extra1( ) ) ) ; + +#$sync->{imap1}->State( 0 ); # Unconnected +if ( ! lost_connection( $sync, $sync->{imap1}, "for host1 [$sync->{host1}]" ) ) { $sync->{imap1}->logout( ) ; } +if ( ! lost_connection( $sync, $sync->{imap2}, "for host2 [$sync->{host2}]" ) ) { $sync->{imap2}->logout( ) ; } if ( $sync->{ nb_errors } ) { @@ -3172,6 +3205,43 @@ sub print_output_if_needed } +sub stderr_to_stdout +{ + my $mysync = shift @ARG ; + if ( $mysync->{ tee} ) + { + *STDERR = *${ $mysync->{ tee } }{ IO } ; + } + else + { + *STDERR = *STDOUT ; + } + return ; +} + + + +sub determine_delete2duplicates +{ + my $mysync = shift @ARG ; + + if ( defined $mysync->{ delete2duplicates } ) + { + return $mysync->{ delete2duplicates } ; + } + + if ( $mysync->{ syncduplicates } ) + { + return 0 ; + } + + if ( $sync->{ delete2 } ) + { + return 1 ; + } + + return ; +} sub define_pidfile { @@ -3803,6 +3873,14 @@ sub buggyflagsregex return( @buggyflagsregex ) ; } + +sub make_var_array_to_a_hash +{ + my $mysync = shift @ARG ; + %{ $mysync->{ varh } } = split_around_equal( @{ $mysync->{ var } } ) ; +} + + sub cgisetcontext { my $mysync = shift ; @@ -3833,7 +3911,7 @@ sub cgisetcontext $mysync->{ errorsmax } = $ERRORS_MAX_CGI ; $modulesversion = 0 ; $mysync->{ releasecheck } = defined $mysync->{ releasecheck } ? $mysync->{ releasecheck } : 1 ; - $usecache = 0 ; + $mysync->{ usecache } = 0 ; $mysync->{ showpasswords } = 0 ; $mysync->{ acc1 }->{ debugimap } = 0 ; $mysync->{ acc2 }->{ debugimap } = 0 ; @@ -3860,7 +3938,7 @@ sub cgisetcontext $mysync->{ logdir } = '' ; chdir $cgidir or die "Can not cd to $cgidir: $OS_ERROR\n" ; - cgioutputenvcontext( $mysync ) ; + output( $mysync, cgienvcontext( $mysync ) ) ; $mysync->{ debug } and output( $mysync, 'Current directory is ' . getcwd( ) . "\n" ) ; $mysync->{ debug } and output( $mysync, 'Real user id is ' . getpwuid_any_os( $REAL_USER_ID ) . " (uid $REAL_USER_ID)\n" ) ; $mysync->{ debug } and output( $mysync, 'Effective user id is ' . getpwuid_any_os( $EFFECTIVE_USER_ID ). " (euid $EFFECTIVE_USER_ID)\n" ) ; @@ -3887,18 +3965,77 @@ sub cgisetcontext return ; } -sub cgioutputenvcontext + + +sub tests_cgienvcontext +{ + note( 'Entering tests_cgienvcontext()' ) ; + + is( '', cgienvcontext( ), 'cgienvcontext: no args => empty' ) ; + my $mysync = { } ; + is( '', cgienvcontext( $mysync ), 'cgienvcontext: undef => empty' ) ; + + # environment SERVER_SOFTWARE alone + local $ENV{SERVER_SOFTWARE} = 'Chateau Lami' ; + is( "SERVER_SOFTWARE is Chateau Lami\n", cgienvcontext( $mysync ), 'cgienvcontext: SERVER_SOFTWARE=Chateau Lami' ) ; + + # environment REMOTE_HOST and SERVER_SOFTWARE + $mysync = { } ; + local $ENV{REMOTE_HOST} = 'Votre serviteur' ; + is( "REMOTE_HOST is Votre serviteur\nSERVER_SOFTWARE is Chateau Lami\n", cgienvcontext( $mysync ), 'cgienvcontext: SERVER_SOFTWARE + REMOTE_HOST' ) ; + + # environment REMOTE_HOST and SERVER_SOFTWARE and --var REMOTE_HOST + $mysync->{ varh }->{REMOTE_HOST} = 'Another Servant' ; + is( "REMOTE_HOST is Another Servant and Votre serviteur\nSERVER_SOFTWARE is Chateau Lami\n", cgienvcontext( $mysync ), 'cgienvcontext: SERVER_SOFTWARE + REMOTE_HOST + --var REMOTE_HOST' ) ; + + + # environment SERVER_SOFTWARE --var REMOTE_HOST only + local $ENV{REMOTE_HOST} = undef ; + $mysync = { } ; + $mysync->{ varh }->{REMOTE_HOST} = 'Another Servant' ; + is( "REMOTE_HOST is Another Servant\nSERVER_SOFTWARE is Chateau Lami\n", cgienvcontext( $mysync ), 'cgienvcontext: SERVER_SOFTWARE + --var REMOTE_HOST' ) ; + + # environment SERVER_SOFTWARE --var REMOTE_HOST only + local $ENV{REMOTE_HOST} = undef ; + local $ENV{SERVER_SOFTWARE} = undef ; + $mysync = { } ; + $mysync->{ varh }->{REMOTE_HOST} = 'Another Servant' ; + is( "REMOTE_HOST is Another Servant\n", cgienvcontext( $mysync ), 'cgienvcontext: --var REMOTE_HOST' ) ; + + note( 'Leaving tests_cgienvcontext()' ) ; + return ; +} + +sub cgienvcontext { my $mysync = shift @ARG ; - - for my $envvar ( qw( REMOTE_ADDR REMOTE_HOST HTTP_REFERER HTTP_USER_AGENT SERVER_SOFTWARE SERVER_PORT HTTP_COOKIE ) ) { - + + if ( ! defined $mysync ) { return '' ; } + + my $output = '' ; + for my $envvar ( qw( REMOTE_ADDR REMOTE_HOST HTTP_REFERER HTTP_USER_AGENT SERVER_SOFTWARE SERVER_NAME SERVER_ADDR SERVER_PORT SERVER_ADMIN HTTP_COOKIE ) ) + { + # $envval comes from the web server via an environment variable + # $varval comes from the command line parameter --val or the cgi parameter val my $envval = $ENV{ $envvar } || q{} ; - if ( $envval ) { output( $mysync, "$envvar is $envval\n" ) } ; + my $varval = $mysync->{ varh }->{ $envvar } ; + + if ( $envval and ! $varval ) + { + $output .= "$envvar is $envval\n" ; + } + elsif ( ! $envval and $varval ) + { + $output .= "$envvar is $varval\n" ; + } + elsif ( $envval and $varval ) + { + $output .= "$envvar is $varval and $envval\n" ; + } } - return ; -} + return $output ; +} sub announcelogfile { @@ -4005,18 +4142,6 @@ sub debugsleep return ; } -sub tests_foldersize -{ - note( 'Entering tests_foldersize()' ) ; - - is( undef, foldersize( ), 'foldersize: no args => undef' ) ; - - - #is_deeply( {}, {}, 'foldersize: a hash is a hash' ) ; - #is_deeply( [], [], 'foldersize: an array is an array' ) ; - note( 'Leaving tests_foldersize()' ) ; - return ; -} @@ -4506,7 +4631,7 @@ sub foldersizes_at_the_beggining myprint( << 'END_SIZE' ) ; -Folders sizes before the synchronization. +Folders sizes before the synchronization. It can take some time. Be patient. You can remove foldersizes listings by using "--nofoldersizes" and "--nofoldersizesatend" but then you will also lose the ETA (Estimation Time of Arrival) given after each message copy. END_SIZE @@ -5418,13 +5543,13 @@ sub imap_id myprint( "$Side: No ID capability\n" ) ; }else{ my $id_inp = imapsync_id( $mysync, { side => lc $Side } ) ; - myprint( "\n$Side: found ID capability. Sending/receiving ID, presented in raw IMAP for now.\n" + myprint( "$Side: found ID capability. Sending/receiving ID, presented in raw IMAP for now.\n" . "In order to avoid sending/receiving ID, use option --noid\n" ) ; my $debug_before = $imap->Debug( ) ; $imap->Debug( 1 ) ; my $id_out = $imap->tag_and_run( 'ID ' . $id_inp ) ; #my $id_out = $imap->tag_and_run( 'ID NIL' ) ; - myprint( "\n" ) ; + #myprint( "\n" ) ; $imap->Debug( $debug_before ) ; #$imap_id_response = Data::Dumper->Dump( [ $id_out ], [ 'IMAP_ID' ] ) ; } @@ -5446,7 +5571,7 @@ sub imapsync_id vendor => 'Gilles LAMIRAL', 'support-url' => 'https://imapsync.lamiral.info/', # Example of date-time: 19-Sep-2015 08:56:07 - date => date_from_rcs( q{$Date: 2022/01/12 21:28:37 $ } ), + date => date_from_rcs( q{$Date: 2022/04/04 16:05:55 $ } ), } ; my $imapsync_id_github = { @@ -5455,7 +5580,7 @@ sub imapsync_id os => $OSNAME, vendor => 'github', 'support-url' => 'https://github.com/imapsync/imapsync', - date => date_from_rcs( q{$Date: 2022/01/12 21:28:37 $ } ), + date => date_from_rcs( q{$Date: 2022/04/04 16:05:55 $ } ), } ; $imapsync_id = $imapsync_id_lamiral ; @@ -5624,6 +5749,15 @@ sub quota_extract_storage_current_in_bytes } + +sub make_f1f2_array_to_a_hash +{ + my $mysync = shift @ARG ; + %{ $mysync->{ f1f2h } } = split_around_equal( @{ $mysync->{ f1f2 } } ) ; +} + + + sub automap { my ( $mysync ) = @_ ; @@ -5776,7 +5910,7 @@ sub build_possible_special $possible_special->{'\Sent'} = [ 'Sent', 'Sent Messages', 'Sent Items', 'Gesendete Elemente', 'Gesendete Objekte', '&AMk-l&AOk-ments envoy&AOk-s', 'E&AwE-le&AwE-ments envoye&AwE-s', 'Envoy&AOk-', 'Objets envoy&AOk-s', - 'Elementos enviados', + 'Elementos enviados', 'Posta inviata', '&kAFP4W4IMH8wojCkMMYw4A-', '&BB4EQgQ,BEAEMAQyBDsENQQ9BD0ESwQ1-', 'Elementy wys&AUI-ane'] ; @@ -6332,9 +6466,9 @@ sub size_filtered_flag return( 0 ) ; } -sub sync_flags_fir +sub sync_flags_fir { - my ( $mysync, $h1_fold, $h1_msg, $h2_fold, $h2_msg, $permanentflags2, $h1_fir_ref, $h2_fir_ref ) = @_ ; + my ( $mysync, $h1_fold, $h1_msg, $h2_fold, $h2_msg, $h1_fir_ref, $h2_fir_ref ) = @_ ; if ( not defined $h1_msg ) { return } ; if ( not defined $h2_msg ) { return } ; @@ -6346,44 +6480,39 @@ sub sync_flags_fir my $h1_flags = $h1_fir_ref->{ $h1_msg }->{ 'FLAGS' } || q{} ; my $h2_flags = $h2_fir_ref->{ $h2_msg }->{ 'FLAGS' } || q{} ; - sync_flags( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg, $h2_flags, $permanentflags2 ) ; + sync_flags( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg, $h2_flags ) ; return ; -} +} -sub sync_flags_after_copy +sub sync_flags_after_copy { # Activated with option --syncflagsaftercopy - my( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg, $permanentflags2 ) = @_ ; + my( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg ) = @_ ; if ( my @h2_flags = $mysync->{imap2}->flags( $h2_msg ) ) { my $h2_flags = "@h2_flags" ; - ( $mysync->{ debug } or $debugflags ) and myprint( "Host2: msg $h2_fold/$h2_msg flags before sync flags after copy ( $h2_flags )\n" ) ; - sync_flags( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg, $h2_flags, $permanentflags2 ) ; + ( $mysync->{ debug } or $mysync->{ debugflags } ) and myprint( "Host2: msg $h2_fold/$h2_msg flags before sync flags after copy ( $h2_flags )\n" ) ; + sync_flags( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg, $h2_flags ) ; }else{ myprint( "Host2: msg $h2_fold/$h2_msg could not get its flags for sync flags after copy\n" ) ; } return ; -} - -# Globals -# $debug -# $debugflags -# $permanentflags2 +} -sub sync_flags +sub sync_flags { - my( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg, $h2_flags, $permanentflags2 ) = @_ ; + my( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $h2_msg, $h2_flags ) = @_ ; - ( $mysync->{ debug } or $debugflags ) and + ( $mysync->{ debug } or $mysync->{ debugflags } ) and myprint( "Host1: flags init msg $h1_fold/$h1_msg flags( $h1_flags ) Host2 msg $h2_fold/$h2_msg flags( $h2_flags )\n" ) ; - $h1_flags = flags_for_host2( $mysync, $h1_flags, $permanentflags2 ) ; + $h1_flags = flags_for_host2( $mysync, $h1_flags, $mysync->{ permanentflags2 } ) ; $h2_flags = flagscase( $h2_flags ) ; - ( $mysync->{ debug } or $debugflags ) and + ( $mysync->{ debug } or $mysync->{ debugflags } ) and myprint( "Host1: flags filt msg $h1_fold/$h1_msg flags( $h1_flags ) Host2 msg $h2_fold/$h2_msg flags( $h2_flags )\n" ) ; @@ -6392,7 +6521,7 @@ sub sync_flags my @h2_flags = sort split(q{ }, $h2_flags ); my $diff = compare_lists( \@h1_flags, \@h2_flags ); - $diff and ( $mysync->{ debug } or $debugflags ) + $diff and ( $mysync->{ debug } or $mysync->{ debugflags } ) and myprint( "Host2: flags msg $h2_fold/$h2_msg replacing h2 flags( $h2_flags ) with h1 flags( $h1_flags )\n" ) ; # This sets flags exactly. So flags can be removed with this. @@ -6400,14 +6529,14 @@ sub sync_flags # to be removed on host2. Just add flags is not what # we need most of the time, so no + like in "+FLAGS.SILENT". - if ( not $mysync->{dry} and $diff and not $mysync->{imap2}->store( $h2_msg, "FLAGS.SILENT (@h1_flags)" ) ) { + if ( not $mysync->{ dry } and $diff and not $mysync->{ imap2 }->store( $h2_msg, "FLAGS.SILENT (@h1_flags)" ) ) { my $error_msg = join q{}, "Host2: flags msg $h2_fold/$h2_msg could not add flags [@h1_flags]: ", - $mysync->{imap2}->LastError || q{}, "\n" ; + $mysync->{ imap2 }->LastError || q{}, "\n" ; errors_incr( $mysync, $error_msg ) ; } return ; -} +} @@ -6916,25 +7045,25 @@ sub remove_tmp_files return ; } -sub cleanup_before_exit +sub cleanup_before_exit { my $mysync = shift ; remove_tmp_files( $mysync ) ; - if ( $mysync->{imap1} and $mysync->{imap1}->IsConnected() ) + if ( $mysync->{ imap1 } and $mysync->{ imap1 }->IsConnected() ) { myprint( "Disconnecting from host1 $mysync->{ host1 } user1 $mysync->{ user1 }\n" ) ; - $mysync->{imap1}->logout( ) ; + $mysync->{ imap1 }->logout( ) ; } - if ( $mysync->{imap2} and $mysync->{imap2}->IsConnected() ) + if ( $mysync->{ imap2 } and $mysync->{ imap2 }->IsConnected() ) { myprint( "Disconnecting from host2 $mysync->{ host2 } user2 $mysync->{ user2 }\n" ) ; - $mysync->{imap2}->logout( ) ; + $mysync->{ imap2 }->logout( ) ; } - if ( $mysync->{log} ) { + if ( $mysync->{ log } ) { myprint( "Log file is $mysync->{logfile} ( to change it, use --logfile filepath ; or use --nolog to turn off logging )\n" ) ; } else @@ -6942,13 +7071,13 @@ sub cleanup_before_exit myprint( "No log file because of option --nolog\n" ) ; } - if ( $mysync->{log} and $mysync->{logfile_handle} ) { + if ( $mysync->{ log } ) { #print( "Closing $mysync->{ logfile }\n" ) ; - teefinish( $mysync ) ; + teefinish( $mysync->{ tee } ) ; } - + $IO::Socket::SSL::DEBUG = 1 ; return ; -} +} sub tests_exit_value @@ -7574,7 +7703,7 @@ sub probe_imapssl } -sub connect_imap +sub connect_imap { my( $host, $port, $ssl, $tls, $acc ) = @_ ; my $imap = Mail::IMAPClient->new( ) ; @@ -7601,8 +7730,7 @@ sub connect_imap " $OS_ERROR\n" ) ; } - myprint( "$acc->{ Side } IP address: ", $imap->Socket->peerhost(), "\n" ) ; - + myprint( "$acc->{ Side } IP address: ", $imap->Socket->peerhost(), " Local IP address: ", $imap->Socket->sockhost(), "\n" ) ; my $banner = $imap->Results()->[0] ; myprint( "$acc->{ Side } banner: $banner" ) ; @@ -7621,7 +7749,7 @@ sub connect_imap myprint( "$acc->{ Side }: Socket successfully converted to SSL\n" ) ; } return( $imap ) ; -} +} sub tests_compress_ssl { @@ -8013,7 +8141,7 @@ sub tests_login_imap_oauth -sub login_imap +sub login_imap { my @allargs = @_ ; my( @@ -8040,7 +8168,9 @@ sub login_imap errors_incr( $mysync, $error ) ; return ; } - myprint( "$acc->{ Side } IP address: ", $imap->Socket->peerhost(), "\n" ) ; + + # Add also $imap->Socket->sockhost() to help configuring firewalls, allowed rule. + myprint( "$acc->{ Side } IP address: ", $imap->Socket->peerhost(), " Local IP address: ", $imap->Socket->sockhost(), "\n" ) ; my $banner = $imap->Results()->[0] ; myprint( "$acc->{ Side } banner: $banner" ) ; @@ -8094,7 +8224,7 @@ sub login_imap myprint( "$acc->{ Side }: failed login on [$host] with user [$user] auth [$acc->{ authmech }]\n" ) ; return ; } -} +} @@ -8109,12 +8239,9 @@ sub init_imap $imap = Mail::IMAPClient->new() ; - if ( $mysync->{ tee } ) - { - # Well, it does not change anything, does it? - # It does when suppressing the hack with *STDERR - $imap->Debug_fh( $mysync->{ tee } ) ; - } + # Well, it does not change anything, does it? + # It does when suppressing the hack with *STDERR + $imap->Debug_fh( $mysync->{ tee } || *STDOUT ) ; if ( $ssl ) { set_ssl( $imap, $acc ) } if ( $tls ) { } # can not do set_tls() here because connect() will directly do a STARTTLS @@ -8671,8 +8798,8 @@ sub banner_imapsync my $banner_imapsync = join q{}, q{$RCSfile: imapsync,v $ }, - q{$Revision: 2.178 $ }, - q{$Date: 2022/01/12 21:28:37 $ }, + q{$Revision: 2.200 $ }, + q{$Date: 2022/04/04 16:05:55 $ }, "\n", "Command line used, run by $EXECUTABLE_NAME:\n", "$PROGRAM_NAME ", command_line_nopassword( $mysync, @argv ), "\n" ; @@ -10693,7 +10820,7 @@ sub tests_regexflags return ; } -sub regexflags +sub regexflags { my $mysync = shift ; my $flags = shift ; @@ -10701,16 +10828,16 @@ sub regexflags foreach my $regexflag ( @{ $mysync->{ regexflag } } ) { my $flags_orig = $flags ; - $debugflags and myprint( "eval \$flags =~ $regexflag\n" ) ; + $mysync->{ debugflags } and myprint( "eval \$flags =~ $regexflag\n" ) ; my $ret = eval "\$flags =~ $regexflag ; 1 " ; - $debugflags and myprint( "regexflag $regexflag [$flags_orig] -> [$flags]\n" ) ; + $mysync->{ debugflags } and myprint( "regexflag $regexflag [$flags_orig] -> [$flags]\n" ) ; if( not ( defined $ret ) or $EVAL_ERROR ) { myprint( "Error: eval regexflag '$regexflag': $EVAL_ERROR\n" ) ; return( undef ) ; } } return( $flags ) ; -} +} sub filterbuggyflags @@ -10830,7 +10957,7 @@ sub tests_permanentflags return ; } -sub permanentflags +sub permanentflags { my $mysync = shift ; @@ -10838,7 +10965,7 @@ sub permanentflags foreach my $line (@lines) { if ( $line =~ m{\[PERMANENTFLAGS\s\(([^)]+?)\)\]}x ) { - ( $debugflags or $sync->{ debug } ) and myprint( "permanentflags: $line" ) ; + ( $mysync->{ debugflags } or $mysync->{ debug } ) and myprint( "permanentflags: $line" ) ; my $permanentflags = $1 ; if ( $permanentflags =~ m{\\\*}x ) { @@ -10848,7 +10975,7 @@ sub permanentflags } ; } return( q{} ) ; -} +} sub tests_flags_filter { @@ -11041,7 +11168,7 @@ sub select_msgs_by_search # Need all messages list to avoid deleting useful cache part # in case of --search or --minage or --maxage - if ( ( defined $msgs_all_hash_ref and $usecache ) + if ( ( defined $msgs_all_hash_ref and $sync->{ usecache } ) or ( not defined $maxage and not defined $minage and not defined $search_cmd ) ) { @@ -11300,11 +11427,11 @@ sub stats_update_skip_message return ; } -sub copy_message +sub copy_message { # copy - my ( $mysync, $h1_msg, $h1_fold, $h2_fold, $h1_fir_ref, $permanentflags2, $cache_dir ) = @_ ; + my ( $mysync, $h1_msg, $h1_fold, $h2_fold, $h1_fir_ref, $cache_dir ) = @_ ; ( $mysync->{ debug } or $mysync->{dry} ) and myprint( "msg $h1_fold/$h1_msg copying to $h2_fold $mysync->{dry_message} " . eta( $mysync ) . "\n" ) ; @@ -11357,12 +11484,12 @@ sub copy_message my $h1_date = date_for_host2( $h1_msg, $h1_idate ) ; - ( $mysync->{ debug } or $debugflags ) and + ( $mysync->{ debug } or $mysync->{ debugflags } ) and myprint( "Host1: flags init msg $h1_fold/$h1_msg date [$h1_date] flags [$h1_flags] size [$h1_size]\n" ) ; - $h1_flags = flags_for_host2( $mysync, $h1_flags, $permanentflags2 ) ; + $h1_flags = flags_for_host2( $mysync, $h1_flags, $mysync->{ permanentflags2 } ) ; - ( $mysync->{ debug } or $debugflags ) and + ( $mysync->{ debug } or $mysync->{ debugflags } ) and myprint( "Host1: flags filt msg $h1_fold/$h1_msg date [$h1_date] flags [$h1_flags] size [$h1_size]\n" ) ; $h1_date = undef if ( $h1_date eq q{} ) ; @@ -11372,13 +11499,13 @@ sub copy_message if ( $new_id and $syncflagsaftercopy ) { - sync_flags_after_copy( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $new_id, $permanentflags2 ) ; + sync_flags_after_copy( $mysync, $h1_fold, $h1_msg, $h1_flags, $h2_fold, $new_id ) ; } myprint( debugmemory( $mysync, " at C3" ) ) ; return $new_id ; -} +} @@ -11670,6 +11797,7 @@ sub tests_message_for_host2 is( undef, $string, q{message_for_host2: --pipemess 'true', value} ) ; } + undef @pipemess ; note( 'Leaving tests_message_for_host2()' ) ; return ; } @@ -12327,7 +12455,7 @@ sub append_message_on_host2 $amount_transferred, $eta ); sleep_if_needed( $mysync ) ; - if ( $usecache and $cacheaftercopy and $new_id =~ m{^\d+$}x ) { + if ( $sync->{ usecache } and $cacheaftercopy and $new_id =~ m{^\d+$}x ) { $debugcache and myprint( "touch $cache_dir/${h1_msg}_$new_id\n" ) ; touch( "$cache_dir/${h1_msg}_$new_id" ) or croak( "Couldn't touch $cache_dir/${h1_msg}_$new_id" ) ; @@ -12741,6 +12869,40 @@ sub tests_time_remaining } + + +sub tests_usecache_and_skipcrossduplicates +{ + note( 'Entering tests_usecache_and_skipcrossduplicates()' ) ; + + is( undef, usecache_and_skipcrossduplicates( ), 'usecache_and_skipcrossduplicates: no args => undef' ) ; + my $mysync = { } ; + is( undef, usecache_and_skipcrossduplicates( $mysync ), 'usecache_and_skipcrossduplicates: undef => undef' ) ; + + $mysync->{ usecache } = $mysync->{ skipcrossduplicates } = 1 ; + is( 1, usecache_and_skipcrossduplicates( $mysync ), 'usecache_and_skipcrossduplicates: usecache=skipcrossduplicates=1 => wrong' ) ; + + note( 'Leaving tests_usecache_and_skipcrossduplicates()' ) ; + return ; +} + +sub usecache_and_skipcrossduplicates +{ + my $mysync = shift @ARG ; + + if ( ! defined $mysync ) { return ; } + + if ( ( $mysync->{ usecache } ) and ( $mysync->{ skipcrossduplicates } ) ) + { + return 1 ; + } + + return ; +} + + + + sub cache_map { my ( $cache_files_ref, $h1_msgs_ref, $h2_msgs_ref ) = @_; @@ -13399,6 +13561,7 @@ sub tests_filter_forbidden_characters note( 'Entering tests_filter_forbidden_characters()' ) ; is( undef , filter_forbidden_characters( ), 'filter_forbidden_characters: no args -> undef' ) ; + is( '' , filter_forbidden_characters( '' ), 'filter_forbidden_characters: empty string -> empty string' ) ; is( 'a_b' , filter_forbidden_characters( 'a_b' ), 'filter_forbidden_characters: a_b -> a_b' ) ; is( 'a_b' , filter_forbidden_characters( 'a*b' ), 'filter_forbidden_characters: a*b -> a_b' ) ; @@ -14876,13 +15039,368 @@ sub do_and_print_stats comment_on_final_diff_in_1_not_in_2( $mysync ) ; comment_on_final_diff_in_2_not_in_1( $mysync ) ; - myprint( "Detected $mysync->{nb_errors} errors\n\n" ) ; + myprint( "Detected $mysync->{nb_errors} errors\n" ) ; myprint( $mysync->{ warn_release }, "\n" ) ; myprint( homepage( ), "\n" ) ; return ; } + +sub tests_email_report_message_id +{ + note( 'Entering tests_email_report_message_id()' ) ; + + local $ENV{TZ} = 'GMT' ; + + like( email_report_message_id( ), + qr{^...._.._.._.._.._.._...__\@imapsync.tk$}xms, + 'email_report_message_id: no args => ...._.._.._.._.._.._...__@imapsync.tk' ) ; + + my $mysync = { } ; + like( email_report_message_id( $mysync ), + qr{^...._.._.._.._.._.._...__\@imapsync.tk$}xms, + 'email_report_message_id: undef => ...._.._.._.._.._.._...__@imapsync.tk' ) ; + + $mysync->{ timestart } = 1357902468.531 ; + like( + email_report_message_id( $mysync ), + qr{^2013_01_11_\d\d_07_48_530__\@imapsync.tk$}xms, + 'email_report_message_id: 1357902468.531 => 2013_01_11_\d\d_07_48_530__@imapsync.tk' ) ; + + $mysync->{ user1 } = 'sarah' ; + $mysync->{ user2 } = 'haras' ; + $mysync->{ timestart } = 1357902468.531 ; + like( + email_report_message_id( $mysync ), + qr{^2013_01_11_\d\d_07_48_530_sarah_haras\@imapsync.tk$}xms, + 'email_report_message_id: 1357902468.531 sarah haras => 2013_01_11_\d\d_07_48_530_sarah_haras@imapsync.tk' ) ; + + $mysync->{ user1 } = 'sar@ah' ; + $mysync->{ user2 } = 'har@as' ; + $mysync->{ timestart } = 1357902468.531 ; + like( + email_report_message_id( $mysync ), + qr{2013_01_11_\d\d_07_48_530_sar_ah_har_as\@imapsync.tk}, + 'email_report_message_id: 1357902468.531 sar@ah har@as => 2013_01_11_\d\d_07_48_530_sar_ah_har_as@imapsync.tk' ) ; + + note( 'Leaving tests_email_report_message_id()' ) ; + return ; +} + + +sub email_report_message_id +{ + my $mysync = shift @ARG ; + + my $time = $mysync->{ timestart } || time ; + my $user1_filtered = $mysync->{ user1 } || '' ; + my $user2_filtered = $mysync->{ user2 } || '' ; + + # Nothing but alphanumeric characters and underscores to replace the others + $user1_filtered =~ s/[^a-zA-Z0-9]/_/g ; + $user2_filtered =~ s/[^a-zA-Z0-9]/_/g ; + + my $message_id = join( '', + year_month_day_hour_min_sec_ms( $time ), + '_', + $user1_filtered, + '_', + $user2_filtered, + '@imapsync.tk', + ) ; + return $message_id ; +} + + +sub tests_date_rfc822 +{ + note( 'Entering tests_email_report_date()' ) ; + + ok( date_rfc822( ), 'date_rfc822: no args => now: ' . date_rfc822( ) ) ; + + if ( 'MSWin32' eq $OSNAME ) + { + like( date_rfc822( 1671706800 ), qr{Thu, 22 Dec 2022 \d\d:00:00 \+0000}, 'date_rfc822: 1671706800 => Thu, 22 Dec 2022 \d\d:00:00 \+0000' ) ; + like( date_rfc822( 1671534000 ), qr{Tue, 20 Dec 2022 \d\d:00:00 \+0000}, 'date_rfc822: 1671534000 => Tue, 20 Dec 2022 \d\d:00:00 \+0000' ) ; + + } + else + { + local $ENV{TZ} = 'GMT' ; + is( 'Thu, 01 Jan 1970 00:00:00 +0000', date_rfc822( 0 ), 'date_rfc822: 0 => Thu, 01 Jan 1970 00:00:00 +0000' ) ; + is( 'Thu, 22 Dec 2022 11:00:00 +0000', date_rfc822( 1671706800 ), 'date_rfc822: 1671706800 => Thu, 22 Dec 2022 11:00:00 +0000' ) ; + } + note( 'Leaving tests_email_report_date()' ) ; + return ; +} + + + +sub date_rfc822 +{ + # Later I found Mail::IMAPCLient::Rfc822_date() + # https://metacpan.org/pod/Mail::IMAPClient#Rfc822_date + my $time = shift @ARG ; + $time = defined( $time ) ? $time : time ; + my $old_locale = POSIX::setlocale( POSIX::LC_TIME, "C" ) ; + + my $date_rfc822 ; + + if ( 'MSWin32' eq $OSNAME ) + { + $date_rfc822 = POSIX::strftime( "%a, %d %b %Y %H:%M:%S +0000", localtime( $time ) ) ; + } + else + { + $date_rfc822 = POSIX::strftime( "%a, %d %b %Y %H:%M:%S %z", localtime( $time ) ) ; + } + + POSIX::setlocale( POSIX::LC_TIME, $old_locale ) ; + return $date_rfc822 ; +} + + + +sub tests_email_report_from +{ + note( 'Entering tests_email_report_from()' ) ; + + is( 'help@imapsync.tk', email_report_from( ), 'email_report_from: no args => help@imapsync.tk' ) ; + my $mysync = { } ; + is( 'help@imapsync.tk', email_report_from( $mysync ), 'email_report_from: undef => help@imapsync.tk' ) ; + $mysync->{ email_report_from } = 'foo@example.com' ; + is( 'foo@example.com', email_report_from( $mysync ), 'email_report_from: foo@example.com => foo@example.com' ) ; + + note( 'Leaving tests_email_report_from()' ) ; + return ; +} + +sub email_report_from +{ + my $mysync = shift @ARG ; + + my $email_report_from = defined( $mysync->{ email_report_from } ) + ? $mysync->{ email_report_from } + : 'help@imapsync.tk' ; + + return $email_report_from ; +} + + +sub tests_email_report_to +{ + note( 'Entering tests_email_report_from()' ) ; + + is( 'unknown@imapsync.tk', email_report_to( ), 'email_report_to: no args => help@imapsync.tk' ) ; + my $mysync = { } ; + is( 'unknown@imapsync.tk', email_report_to( $mysync ), 'email_report_to: undef => help@imapsync.tk' ) ; + $mysync->{ user2 } = 'foo@example.com' ; + is( 'foo@example.com', email_report_to( $mysync ), 'email_report_to: foo@example.com => foo@example.com' ) ; + + note( 'Leaving tests_email_report_from()' ) ; + return ; +} + +sub email_report_to +{ + my $mysync = shift @ARG ; + + my $email_report_to = defined( $mysync->{ user2 } ) + ? $mysync->{ user2 } + : 'unknown@imapsync.tk' ; + + return $email_report_to ; +} + +sub email_report_subject +{ + my $mysync = shift @ARG ; + + my $email_report_subject = 'Imapsync transferred your account' ; + return $email_report_subject ; +} + +sub tests_email_report_body_base +{ + note( 'Entering tests_email_report_body_base()' ) ; + + is( '', email_report_body_base( ), 'email_report_body_base: no args => empty string' ) ; + my $mysync = { } ; + is( '', email_report_body_base( $mysync ), 'email_report_body_base: undef => empty string' ) ; + $mysync->{ user1 } = 'user1@example.com' ; + $mysync->{ user2 } = 'user2@example.com' ; + + note( email_report_body_base( $mysync ) ) ; + + note( 'Leaving tests_email_report_body_base()' ) ; + return ; +} + +sub email_report_body_base +{ + my $mysync = shift @ARG ; + + if( ! $mysync ) { return '' ; } + if( ! all_defined( + $mysync->{ user1 }, + $mysync->{ user2 }, + ) ) { return '' ; } + + my $email_report_body_base = <<"EOM"; +\r +\r +\r +Imapsync transfer from $mysync->{ user1 } to $mysync->{ user2 }\r +\r +\r +

    Hello!

    \r +

    \r +Imapsync just ended the synchronization from the imap account $mysync->{ user1 } to the imap account $mysync->{ user2 }.
    \r +

    \r +EOM + return $email_report_body_base ; +} + +sub email_report_body_extra1 +{ + my $image = '' + .'imapsync website' + .'' + . "\r\n" + ; + return $image ; +} + + +sub email_report_body_extra2 +{ + my $image = '' + .'imapsync website' + .'' + . "\r\n" + ; + return $image ; +} + + + + + +sub email_report_html_begin +{ + return '' + . "\r\n" + ; +} + +sub email_report_html_end +{ + return '' + . "\r\n" + ; +} + +sub email_report_body_begin +{ + return '' + . "\r\n" + ; +} + +sub email_report_body_end +{ + return '' + . "\r\n" + ; +} + + +sub email_report_header +{ + my $mysync = shift @ARG ; + + my @header ; + + push( @header, join( "\r\n", + 'Message-Id: ' . email_report_message_id( $mysync ), + 'Date: ' . date_rfc822( time ), + 'From: ' . email_report_from( $mysync ), + 'To: ' . email_report_to( $mysync ), + 'Subject: ' . email_report_subject( $mysync ), + 'Content-Type: text/html', + "\r\n", + ) + ) ; + + return @header ; +} + + +sub tests_email_report +{ + note( 'Entering tests_email_report()' ) ; + + is( '', email_report( ), 'email_report: undef => empty string' ) ; + my $mysync = { } ; + $mysync->{ user1 } = 'user1@example.com' ; + $mysync->{ user2 } = 'user2@example.com' ; + + note( email_report( $mysync, "c'est extra !\r\n" ) ) ; + + note( 'Leaving tests_email_report()' ) ; + return ; +} + + +sub email_report +{ + my $mysync = shift @ARG ; + + my $extra = shift @ARG ; + + if ( ! defined $mysync ) { return '' ; } + + my @email_report = ( ) ; + + push( @email_report, email_report_header( $mysync ), + email_report_html_begin( ), + email_report_body_begin( ), + email_report_body_base( $mysync ), + $extra, + email_report_body_end( ), + email_report_html_end( ), + ) ; + + my $email_report = join( "", @email_report ) ; + + return $email_report ; +} + +sub email_report_append +{ + my $acc = shift ; + my $text = shift ; + + if ( $acc->{ imap }->IsAuthenticated && $text ) + { + my $newuid = $acc->{ imap }->append_string( 'INBOX', $text ) ; + if ( $newuid ) + { + myprint( "$acc->{ Side }: Successfully put the email final report in INBOX. Use --noemailreport" . $acc->{N} . " to avoid it.\n" ) ; + } + else + { + myprint( "$acc->{ Side }: Failed to put the email final report in INBOX. Use --noemailreport" . $acc->{N} . " to avoid it.\n" ) ; + } + return $newuid ; + } + else + { + return ; + } +} + sub diff_or_NA { my( $n1, $n2 ) = @ARG ; @@ -14983,14 +15501,14 @@ sub load_modules # Globals: $skipsize $wholeheaderifneeded sub parse_header_msg { - my ( $mysync, $imap, $m_uid, $s_heads, $s_fir, $side, $s_hash ) = @_ ; + my ( $mysync, $imap, $m_uid, $folder, $s_heads, $s_fir, $side, $s_hash ) = @_ ; my $head = $s_heads->{$m_uid} ; my $headnum = scalar keys %{ $head } ; - $mysync->{ debug } and myprint( "$side: uid $m_uid number of headers, pass one: ", $headnum, "\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid number of headers, pass one: ", $headnum, "\n" ) ; if ( ( ! $headnum ) and ( $wholeheaderifneeded ) ){ - $mysync->{ debug } and myprint( "$side: uid $m_uid no header by parse_headers so taking whole header with BODY.PEEK[HEADER]\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid no header by parse_headers so taking whole header with BODY.PEEK[HEADER]\n" ) ; $imap->fetch($m_uid, 'BODY.PEEK[HEADER]' ) ; my $whole_header = $imap->_transaction_literals ; @@ -14998,16 +15516,16 @@ sub parse_header_msg $head = decompose_header( $whole_header ) ; $headnum = scalar keys %{ $head } ; - $mysync->{ debug } and myprint( "$side: uid $m_uid number of headers, pass two: ", $headnum, "\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid number of headers, pass two: ", $headnum, "\n" ) ; } #myprint( Data::Dumper->Dump( [ $head, \%useheader ] ) ) ; - my $headstr = header_construct( $mysync, $head, $side, $m_uid ) ; + my $headstr = header_construct( $mysync, $head, $side, $folder, $m_uid ) ; if ( ( ! $headstr ) and ( $mysync->{addheader} ) and ( $side eq 'Host1' ) ) { my $header = add_header( $m_uid ) ; - $mysync->{ debug } and myprint( "$side: uid $m_uid no header found so adding our own [$header]\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid no header found so adding our own [$header]\n" ) ; $headstr .= uc $header ; $s_fir->{$m_uid}->{NO_HEADER} = 1; } @@ -15035,7 +15553,7 @@ sub parse_header_msg my $num = scalar( @{ $dup_ref } ) ; push( @{ $dup_ref }, $m_uid ) ; my $keydup = "$key#$num" ; - $mysync->{ debug } and myprint( "$side: uid $m_uid sig $keydup size $size idate $idate dup @{ $dup_ref }\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid sig $keydup size $size idate $idate dup @{ $dup_ref }\n" ) ; if ( $mysync->{ syncduplicates } ) { $s_hash->{"$keydup"}{'5'} = $m_md5 ; @@ -15054,7 +15572,7 @@ sub parse_header_msg $s_hash->{"$key"}{'F'} = $flags ; $s_hash->{"$key"}{'m'} = $m_uid ; $s_hash->{"$key"}{'U'} = [ $m_uid ] ; # ? or [ ] ? - $mysync->{ debug } and myprint( "$side: uid $m_uid sig $key size $size idate $idate\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid sig $key size $size idate $idate\n" ) ; return( 1 ) ; } @@ -15062,7 +15580,7 @@ sub parse_header_msg return ; } -sub tests_header_construct +sub tests_header_construct { note( 'Entering tests_header_construct()' ) ; @@ -15071,39 +15589,39 @@ sub tests_header_construct my $head = { 'key1' => [ 'val1_key1' ] } ; - is( undef, header_construct( $mysync, $head, 'Host1', '1' ), 'header_construct: key1 val1_key1 no useheader => undef' ) ; + is( undef, header_construct( $mysync, $head, 'Host1', 'INBOX', '1' ), 'header_construct: key1 val1_key1 no useheader => undef' ) ; $mysync->{useheader}->{ 'KEY1' } = 1 ; - is( 'KEY1: VAL1_KEY1', header_construct( $mysync, $head, 'Host1', '1' ), 'header_construct: key1 val1_key1 => KEY1: VAL1_KEY1' ) ; + is( 'KEY1: VAL1_KEY1', header_construct( $mysync, $head, 'Host1', 'INBOX', '1' ), 'header_construct: key1 val1_key1 => KEY1: VAL1_KEY1' ) ; $head = { 'key1' => [ 'val1_key1', 'val3_key1', 'val2_key1' ] } ; - is( 'KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1', header_construct( $mysync, $head, 'Host1', '1' ), + is( 'KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1', header_construct( $mysync, $head, 'Host1', 'INBOX', '1' ), 'header_construct: key1 val1_key1 val3_key1 val2_key1 => KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1' ) ; $head = { 'key1' => [ 'val1_key1', 'val3_key1', ' val2_key1' ] } ; - is( 'KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1', header_construct( $mysync, $head, 'Host1', '1' ), + is( 'KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1', header_construct( $mysync, $head, 'Host1', 'INBOX', '1' ), 'header_construct: key1 val1_key1 val3_key1 val2_key1 => KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1' ) ; $mysync->{useheader}->{ 'ALL' } = 1 ; - is( 'KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1', header_construct( $mysync, $head, 'Host1', '1' ), + is( 'KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1', header_construct( $mysync, $head, 'Host1', 'INBOX', '1' ), 'header_construct: key1 val1_key1 val3_key1 val2_key1 useheader ALL => KEY1: VAL1_KEY1KEY1: VAL2_KEY1KEY1: VAL3_KEY1' ) ; $mysync->{skipheader} = 'key1' ; - is( undef, header_construct( $mysync, $head, 'Host1', '1' ), + is( undef, header_construct( $mysync, $head, 'Host1', 'INBOX', '1' ), 'header_construct: key1 val1_key1 val3_key1 val2_key1 useheader ALL => undef' ) ; $head = { 'key1' => [ 'val1_key1', 'val3_key1', ' val2_key1' ], 'key2' => [ 'val1_key2', 'val3_key2', ' val2_key2' ] } ; - is( 'KEY2: VAL1_KEY2KEY2: VAL2_KEY2KEY2: VAL3_KEY2', header_construct( $mysync, $head, 'Host1', '1' ), + is( 'KEY2: VAL1_KEY2KEY2: VAL2_KEY2KEY2: VAL3_KEY2', header_construct( $mysync, $head, 'Host1', 'INBOX', '1' ), 'header_construct: ... useheader ALL skipheader key1 => KEY2: VAL1_KEY2KEY2: VAL2_KEY2KEY2: VAL3_KEY2' ) ; @@ -15113,9 +15631,9 @@ sub tests_header_construct # No global in header_construct -sub header_construct +sub header_construct { - my( $mysync, $head, $side, $m_uid ) = @_ ; + my( $mysync, $head, $side, $folder, $m_uid ) = @_ ; my @headstr ; foreach my $h ( sort keys %{ $head } ) { @@ -15127,10 +15645,10 @@ sub header_construct my $H = header_line_normalize( $h, $val ) ; # show stuff in debug mode - $mysync->{ debug } and myprint( "$side uid $m_uid header [$H]", "\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid header [$H]", "\n" ) ; if ( $mysync->{skipheader} and $H =~ m/$mysync->{skipheader}/xi) { - $mysync->{ debug } and myprint( "$side uid $m_uid skipping header [$H]\n" ) ; + $mysync->{ debug } and myprint( "$side: folder/uid $folder/$m_uid skipping header [$H]\n" ) ; next ; } push @headstr, $H ; @@ -16239,8 +16757,8 @@ sub cpu_time my $cpu_time = 0 ; # last element is the sum of all elements $cpu_time = ( map { $cpu_time += $_ } @cpu_times )[ -1 ] ; - my $cpu_time_round = mysprintf( '%.2f', $cpu_time ) ; - $mysync->{ debug } and myprint( join(' + ', @cpu_times), " = $cpu_time ~ $cpu_time_round\n" ) ; + my $cpu_time_rounded = mysprintf( '%.2f', $cpu_time ) ; + $mysync->{ debug } and myprint( join(' + ', @cpu_times), " = $cpu_time ~ $cpu_time_rounded\n" ) ; return $cpu_time ; } @@ -17775,15 +18293,14 @@ sub set_checknoabletosearch } -sub tests_setlogfile +sub tests_setlogfile { note( 'Entering tests_setlogfile()' ) ; my $mysync = {} ; - $mysync->{logdir} = 'vallogdir' ; - $mysync->{logfile} = 'vallogfile.txt' ; - is( 'vallogdir/vallogfile.txt', setlogfile( $mysync ), - 'setlogfile: logdir vallogdir, logfile vallogfile.txt, vallogdir/vallogfile.txt' ) ; + $mysync->{ logdir } = 'vallogdir' ; + is( 'vallogdir/vallogfile.txt', setlogfile( $mysync, 'vallogfile.txt' ), + 'setlogfile: logdir vallogdir, vallogfile.txt => vallogdir/vallogfile.txt' ) ; SKIP: { skip( 'Too hard to have a well known timezone on Windows', 9 ) if ( 'MSWin32' eq $OSNAME ) ; @@ -17794,8 +18311,8 @@ sub tests_setlogfile timestart => 2, } ; - is( "$DEFAULT_LOGDIR/1970_01_01_00_00_02_000__.txt", setlogfile( $mysync ), - "setlogfile: default is like $DEFAULT_LOGDIR/1970_01_01_00_00_02_000__.txt" ) ; + is( '1970_01_01_00_00_02_000__.txt', setlogfile( $mysync ), + 'setlogfile: default is like 1970_01_01_00_00_02_000__.txt' ) ; $mysync = { timestart => 2, @@ -17804,29 +18321,27 @@ sub tests_setlogfile abort => 1, } ; - is( "$DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2_abort.txt", setlogfile( $mysync ), - "setlogfile: default is like $DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2_abort.txt" ) ; + is( '1970_01_01_00_00_02_000_user1_user2_abort.txt', setlogfile( $mysync ), + 'setlogfile: default abort is like 1970_01_01_00_00_02_000_user1_user2_abort.txt' ) ; $mysync = { timestart => 2, user1 => 'user1', user2 => 'user2', - remote => 'zzz', } ; - is( "$DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2_remote.txt", setlogfile( $mysync ), - "setlogfile: default is like $DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2_remote.txt" ) ; + is( '1970_01_01_00_00_02_000_user1_user2_remote.txt', setlogfile( $mysync, undef, '_remote' ), + 'setlogfile: default with _remote is like 1970_01_01_00_00_02_000_user1_user2_remote.txt' ) ; $mysync = { timestart => 2, user1 => 'user1', user2 => 'user2', - remote => 'zzz', abort => 1, } ; - is( "$DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2_remote_abort.txt", setlogfile( $mysync ), - "setlogfile: default is like $DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2_remote_abort.txt" ) ; + is( '1970_01_01_00_00_02_000_user1_user2_remote_abort.txt', setlogfile( $mysync, undef, '_remote' ), + 'setlogfile: default abort with _remote is like 1970_01_01_00_00_02_000_user1_user2_remote_abort.txt' ) ; $mysync = { @@ -17835,13 +18350,13 @@ sub tests_setlogfile user2 => 'user2', } ; - is( "$DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2.txt", setlogfile( $mysync ), - "setlogfile: default is like $DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2.txt" ) ; + is( '1970_01_01_00_00_02_000_user1_user2.txt', setlogfile( $mysync ), + 'setlogfile: default is like 1970_01_01_00_00_02_000_user1_user2.txt' ) ; $mysync->{logdir} = undef ; $mysync->{logfile} = undef ; - is( "$DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2.txt", setlogfile( $mysync ), - "setlogfile: logdir undef, $DEFAULT_LOGDIR/1970_01_01_00_00_02_000_user1_user2.txt" ) ; + is( '1970_01_01_00_00_02_000_user1_user2.txt', setlogfile( $mysync ), + 'setlogfile: logdir undef, 1970_01_01_00_00_02_000_user1_user2.txt' ) ; $mysync->{logdir} = q{} ; $mysync->{logfile} = undef ; @@ -17858,43 +18373,117 @@ sub tests_setlogfile user2 => 'u/ser2a*|?:"<>b', } ; - is( "$DEFAULT_LOGDIR/1970_01_01_00_00_00_000_us_er1a_______b_u_ser2a_______b.txt", setlogfile( $mysync ), - "setlogfile: logdir undef, $DEFAULT_LOGDIR/1970_01_01_00_00_00_000_us_er1a_______b_u_ser2a_______b.txt" ) ; - - + is( '1970_01_01_00_00_00_000_us_er1a_______b_u_ser2a_______b.txt', setlogfile( $mysync ), + 'setlogfile: logdir undef, 1970_01_01_00_00_00_000_us_er1a_______b_u_ser2a_______b.txt' ) ; } ; note( 'Leaving tests_setlogfile()' ) ; return ; -} +} -sub setlogfile + +sub setlogfile { - my( $mysync ) = shift ; + my $mysync = shift @ARG ; + my $given = shift @ARG ; + my $supplement = shift @ARG || '' ; # When aborting another process the log file name finishes with "_abort.txt" my $abort_suffix = ( $mysync->{ abort } ) ? '_abort' : q{} ; - # When acting as a proxy the log file name finishes with "_remote.txt" - # proxy mode is not done in imapsync, it is done by proximapsync - my $remote_suffix = ( $mysync->{ remote } ) ? '_remote' : q{} ; + my $suffix = logfilesuffix( $mysync, $supplement . $abort_suffix ) ; + my $logdir = $mysync->{ logdir } || '' ; + + my $logfile ; + if ( defined $given ) + { + if ( $logdir ) + { + $logfile = "$logdir/$given" ; + } + else + { + $logfile = "$given" ; + } + } + else + { + $logfile = logfile( $mysync->{ timestart }, $suffix, $logdir ) ; + } + + return( $logfile ) ; +} + + +sub tests_logfilesuffix +{ + note( 'Entering tests_logfilesuffix()' ) ; + + is( '_', logfilesuffix( ), 'logfilesuffix: no args => _' ) ; + my $mysync = { } ; + is( '_', logfilesuffix( $mysync ), 'logfilesuffix: undef => _' ) ; + $mysync->{ user1 } = 'valuser1' ; + $mysync->{ user2 } = 'valuser2' ; + is( 'valuser1_valuser2', logfilesuffix( $mysync ), 'logfilesuffix: valuser1 valuser2 => valuser1_valuser2' ) ; + is( 'valuser1_valuser2_suppl', logfilesuffix( $mysync, '_suppl' ), 'logfilesuffix: valuser1 valuser2 _suppl => valuser1_valuser2_suppl' ) ; + + note( 'Leaving tests_logfilesuffix()' ) ; + return ; +} + +sub logfilesuffix +{ + my $mysync = shift @ARG ; + my $supplement = shift @ARG || '' ; + my $suffix = ( filter_forbidden_characters( slash_to_underscore( $mysync->{ user1 } ) ) || q{} ) . '_' . ( filter_forbidden_characters( slash_to_underscore( $mysync->{ user2 } ) ) || q{} ) - . $remote_suffix . $abort_suffix ; - - $mysync->{ logdir } = defined $mysync->{ logdir } ? $mysync->{ logdir } : $DEFAULT_LOGDIR ; - - $mysync->{ logfile } = defined $mysync->{ logfile } - ? "$mysync->{ logdir }/$mysync->{ logfile }" - : logfile( $mysync->{ timestart }, $suffix, $mysync->{ logdir } ) ; - - return( $mysync->{ logfile } ) ; + . $supplement ; + + return $suffix ; } + + +sub tests_setlogdir +{ + note( 'Entering tests_setlogdir()' ) ; + + is( $DEFAULT_LOGDIR, setlogdir( ), "setlogdir: no args => $DEFAULT_LOGDIR" ) ; + + my $mysync = { } ; + is( $DEFAULT_LOGDIR, setlogdir( $mysync ), "setlogdir: no args => $DEFAULT_LOGDIR" ) ; + + $mysync->{ logdir } = '' ; + is( '', setlogdir( $mysync ), "setlogdir: logdir empty string => empty string" ) ; + is( '', $mysync->{ logdir }, "setlogdir: logdir empty string unchanged" ) ; + + $mysync->{ logdir } = 'vallogdir' ; + is( 'vallogdir', setlogdir( $mysync ), "setlogdir: logdir vallogdir => vallogdir" ) ; + is( 'vallogdir', $mysync->{ logdir }, "setlogdir: logdir vallogdir unchanged" ) ; + # Does a second call hurt? + is( 'vallogdir', setlogdir( $mysync ), "setlogdir: logdir vallogdir => vallogdir" ) ; + is( 'vallogdir', $mysync->{ logdir }, "setlogdir: logdir vallogdir unchanged" ) ; + + note( 'Leaving tests_setlogdir()' ) ; + return ; +} + + +sub setlogdir +{ + my $mysync = shift @ARG ; + + my $logdir = defined $mysync->{ logdir } ? $mysync->{ logdir } : $DEFAULT_LOGDIR ; + + return $logdir ; +} + + sub tests_logfile { note( 'Entering tests_logfile()' ) ; @@ -17906,14 +18495,14 @@ sub tests_logfile local $ENV{TZ} = 'GMT' ; { POSIX::tzset unless ('MSWin32' eq $OSNAME) ; - is( '1970_01_01_00_00_00_000.txt', logfile( ), 'logfile: no args => 1970_01_01_00_00_00.txt' ) ; - is( '1970_01_01_00_00_00_000.txt', logfile( 0 ), 'logfile: 0 => 1970_01_01_00_00_00.txt' ) ; - is( '1970_01_01_00_01_01_000.txt', logfile( 61 ), 'logfile: 0 => 1970_01_01_00_01_01.txt' ) ; - is( '1970_01_01_00_01_01_234.txt', logfile( 61.234 ), 'logfile: 0 => 1970_01_01_00_01_01.txt' ) ; - is( '2010_08_24_14_00_00_000.txt', logfile( 1_282_658_400 ), 'logfile: 1_282_658_400 => 2010_08_24_14_00_00.txt' ) ; - is( '2010_08_24_14_01_01_000.txt', logfile( 1_282_658_461 ), 'logfile: 1_282_658_461 => 2010_08_24_14_01_01.txt' ) ; - is( '2010_08_24_14_01_01_000_poupinette.txt', logfile( 1_282_658_461, 'poupinette' ), 'logfile: 1_282_658_461 poupinette => 2010_08_24_14_01_01_poupinette.txt' ) ; - is( '2010_08_24_14_01_01_000_removeblanks.txt', logfile( 1_282_658_461, ' remove blanks ' ), 'logfile: 1_282_658_461 remove blanks => 2010_08_24_14_01_01_000_removeblanks' ) ; + is( '1970_01_01_00_00_00_000.txt', logfile( ), 'logfile: no args => 1970_01_01_00_00_00_000.txt' ) ; + is( '1970_01_01_00_00_00_000.txt', logfile( 0 ), 'logfile: 0 => 1970_01_01_00_00_00_000.txt' ) ; + is( '1970_01_01_00_01_01_000.txt', logfile( 61 ), 'logfile: 0 => 1970_01_01_00_01_01_000.txt' ) ; + is( '1970_01_01_00_01_01_234.txt', logfile( 61.234 ), 'logfile: 0 => 1970_01_01_00_01_01_234.txt' ) ; + is( '2010_08_24_14_00_00_000.txt', logfile( 1_282_658_400 ), 'logfile: 1_282_658_400 => 2010_08_24_14_00_00_000.txt' ) ; + is( '2010_08_24_14_01_01_000.txt', logfile( 1_282_658_461 ), 'logfile: 1_282_658_461 => 2010_08_24_14_01_01_000.txt' ) ; + is( '2010_08_24_14_01_01_000_poupinette.txt', logfile( 1_282_658_461, 'poupinette' ), 'logfile: 1_282_658_461 poupinette => 2010_08_24_14_01_01_000_poupinette.txt' ) ; + is( '2010_08_24_14_01_01_000_removeblanks.txt', logfile( 1_282_658_461, ' remove blanks ' ), 'logfile: 1_282_658_461 remove blanks => 2010_08_24_14_01_01_000_removeblanks.txt' ) ; is( '2010_08_24_14_01_01_234_poup.txt', logfile( 1_282_658_461.2347, 'poup' ), 'logfile: 1_282_658_461.2347 poup => 2010_08_24_14_01_01_234_poup.txt' ) ; @@ -17921,8 +18510,6 @@ sub tests_logfile is( 'dirdir/2010_08_24_14_01_01_234_poup.txt', logfile( 1_282_658_461.2347, 'poup', 'dirdir' ), 'logfile: 1_282_658_461.2347 poup dirdir => dirdir/2010_08_24_14_01_01_234_poup.txt' ) ; - - } POSIX::tzset unless ('MSWin32' eq $OSNAME) ; } ; @@ -17932,7 +18519,7 @@ sub tests_logfile } -sub logfile +sub logfile { my ( $time, $suffix, $dir ) = @_ ; @@ -17943,45 +18530,146 @@ sub logfile $dir ||= q{} ; my $sep_dir = ( $dir ) ? '/' : q{} ; - my $date_str = POSIX::strftime( '%Y_%m_%d_%H_%M_%S', localtime $time ) ; - # Because of ab tests or web accesses, more than one sync withing one second is possible - # so we add also milliseconds - $date_str .= sprintf "_%03d", ($time - int( $time ) ) * 1000 ; # without rounding + my $date_str = year_month_day_hour_min_sec_ms( $time ) ; + my $logfile = "${dir}${sep_dir}${date_str}${sep_suffix}${suffix}.txt" ; return( $logfile ) ; } -sub tests_localtimez -{ - note( 'Entering tests_localtimez()' ) ; - SKIP: { - # Too hard to have a well known timezone on Windows - skip( 'Too hard to have a well known timezone on Windows', 1 ) if ( 'MSWin32' eq $OSNAME ) ; - local $ENV{TZ} = 'GMT' ; - like( localtimez( 0 ), qr'1970-01-01 00:00:00 \+0000 (GMT|UTC)', 'localtimez: 0 => match 1970-01-01 00:00:00 +0000 GMT' ) ; +sub tests_year_month_day_hour_min_sec_ms +{ + note( 'Entering tests_date_year_month_day_hour_min_sec_ms()' ) ; + + if ( 'MSWin32' eq $OSNAME ) + { + # Can not use $ENV{TZ} nor POSIX::tzset + like( year_month_day_hour_min_sec_ms( ), qr'1970_01_01_\d\d_00_00_000', 'year_month_day_hour_min_sec_ms: no args => match 1970_01_01_\d\d_00_00_000' ) ; + like( year_month_day_hour_min_sec_ms( 0 ), qr'1970_01_01_\d\d_00_00_000', 'year_month_day_hour_min_sec_ms: 0 => match 1970_01_01_\d\d_00_00_000' ) ; + like( year_month_day_hour_min_sec_ms( 1671706800.123 ), qr'2022_12_22_\d\d_00_00_122', 'year_month_day_hour_min_sec_ms: 123456789.123 => match 2022_12_22_\d\d_00_00_122' ) ; + + like( year_month_day_hour_min_sec_ms( -1 ), qr'19\d\d_\d\d_\d\d_\d\d_59_59_000', 'year_month_day_hour_min_sec_ms: -1 => 19\d\d_\d\d_\d\d_\d\d_59_59_000' ) ; + like( year_month_day_hour_min_sec_ms( -0.246 ), qr'19\d\d_\d\d_\d\d_\d\d_59_59_754', 'year_month_day_hour_min_sec_ms: -1 => 19\d\d_\d\d_\d\d_\d\d_59_59_754' ) ; + + like( year_month_day_hour_min_sec_ms( -32360400.135 ), qr'1968_12_22_\d\d_59_59_864', 'year_month_day_hour_min_sec_ms: -1 => 1968_12_22_\d\d_59_59_864' ) ; } + else + { + local $ENV{TZ} = 'GMT' ; + { + POSIX::tzset unless ('MSWin32' eq $OSNAME) ; + is( '1970_01_01_00_00_00_000', year_month_day_hour_min_sec_ms( ), + 'year_month_day_hour_min_sec_ms: no args => 1970_01_01_00_00_00_000 GMT' ) ; + is( '1970_01_01_00_00_00_000', year_month_day_hour_min_sec_ms( 0 ), + 'year_month_day_hour_min_sec_ms: 0 => 1970_01_01_00_00_00_000 GMT' ) ; + is( '1973_11_29_21_33_09_122', year_month_day_hour_min_sec_ms( 123456789.123 ), + 'year_month_day_hour_min_sec_ms: 123456789.123 => 1973_11_29_21_33_09_122 GMT' ) ; + is( '1969_12_31_23_59_59_000', year_month_day_hour_min_sec_ms( -1 ), + 'year_month_day_hour_min_sec_ms: -1 => 1969_12_31_23_59_59_000 GMT' ) ; + is( '1969_12_31_23_59_59_754', year_month_day_hour_min_sec_ms( -0.246 ), + 'year_month_day_hour_min_sec_ms: -0.246 => 1969_12_31_23_59_59_754 GMT' ) ; + is( '1966_02_02_02_26_50_864', year_month_day_hour_min_sec_ms( -123456789.135 ), + 'year_month_day_hour_min_sec_ms: -123456789.135 => 1966_02_02_02_26_50_864 GMT' ) ; + } + POSIX::tzset unless ('MSWin32' eq $OSNAME) ; + } + note( 'Leaving tests_year_month_day_hour_min_sec_ms()' ) ; + return ; +} - is( localtimez( ), localtimez( time ), 'localtimez: undef => equals currrent' ) ; - note( 'Leaving tests_localtimez()' ) ; +sub year_month_day_hour_min_sec_ms +{ + my $time = shift @ARG || 0 ; + + my $date_str = POSIX::strftime( '%Y_%m_%d_%H_%M_%S', localtime $time ) ; + # Because of ab tests or web accesses, more than one sync withing one second is possible + # so we add also milliseconds + $date_str .= sprintf "_%03d", fractional_of_floor( $time ) * 1000 ; + + return $date_str ; +} + + + +sub tests_fractional_of_floor +{ + note( 'Entering tests_fractional_of_floor()' ) ; + + is( 0, fractional_of_floor( ), 'fractional_of_floor: no args => 0' ) ; + is( 0, fractional_of_floor( 0 ), 'fractional_of_floor: 0 => 0' ) ; + is( 0, fractional_of_floor( '0' ), 'fractional_of_floor: 0 => 0' ) ; + is( 0, fractional_of_floor( 1 ), 'fractional_of_floor: 1 => 0' ) ; + is( 0, fractional_of_floor( '1' ), 'fractional_of_floor: 1 => 0' ) ; + is( 0, fractional_of_floor( -1 ), 'fractional_of_floor: -1 => 0' ) ; + is( 0, fractional_of_floor( '-1' ), 'fractional_of_floor: -1 => 0' ) ; + is( 0.234, fractional_of_floor( 1.234 ), 'fractional_of_floor: 1.234 => 0.234' ) ; + is( 0.234, fractional_of_floor( '1.234' ), 'fractional_of_floor: 1.234 => 0.234' ) ; + is( 0.766, fractional_of_floor( -1.234 ), 'fractional_of_floor: -1.234 => 0.766' ) ; + is( 0.766, fractional_of_floor( '-1.234' ), 'fractional_of_floor: -1.234 => 0.766' ) ; + is( 0.234, fractional_of_floor( 10.234 ), 'fractional_of_floor: 10.234 => 0.234' ) ; + is( 0.766, fractional_of_floor( -10.234 ), 'fractional_of_floor: -10.234 => 0.766' ) ; + + note( 'Leaving tests_fractional_of_floor()' ) ; return ; } -sub localtimez + +sub fractional_of_floor +{ + my $float = shift @ARG || 0 ; + + if ( $float - int( $float ) >= 0 ) + { + return( $float - int( $float ) ) ; + } + else + { + return( 1 - ( int( $float ) - $float ) ) ; + } +} + + + +sub tests_localtimez +{ + note( 'Entering tests_localtimez()' ) ; + note( "localtimez: " . localtimez( ) ) ; + if ( 'MSWin32' eq $OSNAME ) + { + like( localtimez( 0 ), qr'1970-01-01 \d\d:\d\d:\d\d', 'localtimez: 0 => match 1970-01-01 \d\d:\d\d:\d\d' ) ; + } + else + { + local $ENV{TZ} = 'GMT' ; + like( localtimez( 0 ), qr'1970-01-01 00:00:00 \+0000 (GMT|UTC)', 'localtimez: 0 => match 1970-01-01 00:00:00 +0000 GMT or UTC' ) ; + } + is( localtimez( ), localtimez( time ), 'localtimez: undef => equals currrent' ) ; + note( 'Leaving tests_localtimez()' ) ; + return ; +} + +sub localtimez { my $time = shift ; $time = defined( $time ) ? $time : time ; - my $datetimestr = POSIX::strftime( '%A %e %B %Y-%m-%d %H:%M:%S %z %Z', localtime( $time ) ) ; - + my $datetimestr ; + + if ( 'MSWin32' eq $OSNAME ) + { + $datetimestr = POSIX::strftime( '%A %d %B %Y-%m-%d %H:%M:%S %z', localtime( $time ) ) ; + } + else + { + $datetimestr = POSIX::strftime( '%A %d %B %Y-%m-%d %H:%M:%S %z %Z', localtime( $time ) ) ; + } #myprint( "$datetimestr\n" ) ; return $datetimestr ; -} - +} @@ -18080,19 +18768,18 @@ sub tests_teelaunch is( undef, teelaunch( ), 'teelaunch: no args => undef' ) ; my $mysync = {} ; is( undef, teelaunch( $mysync ), 'teelaunch: arg empty {} => undef' ) ; - $mysync->{logfile} = q{} ; - is( undef, teelaunch( $mysync ), 'teelaunch: logfile empty string => undef' ) ; + is( undef, teelaunch( $mysync, '' ), 'teelaunch: empty string => undef' ) ; # First time, learning IO::Tee intrasics - $mysync->{logfile} = 'W/tmp/tests/tests_teelaunch.txt' ; - isa_ok( my $tee = teelaunch( $mysync ), 'IO::Tee' , 'teelaunch: logfile W/tmp/tests/tests_teelaunch.txt' ) ; + my $tee = teelaunch( $mysync, 'W/tmp/tests/tests_teelaunch.txt' ) ; + isa_ok( $tee, 'IO::Tee', 'teelaunch: logfile W/tmp/tests/tests_teelaunch.txt' ) ; is( 1, print( $tee "Hi!\n" ), 'teelaunch: write Hi!') ; is( "Hi!\n", file_to_string( 'W/tmp/tests/tests_teelaunch.txt' ), 'teelaunch: reading W/tmp/tests/tests_teelaunch.txt is Hi!\n' ) ; is( 1, print( $tee "Hoo\n" ), 'teelaunch: write Hoo') ; is( "Hi!\nHoo\n", file_to_string( 'W/tmp/tests/tests_teelaunch.txt' ), 'teelaunch: reading W/tmp/tests/tests_teelaunch.txt is Hi!\nHoo\n' ) ; - # closing so tee won't be happy - close $mysync->{logfile_handle} ; + # closing file handle so tee won't be happy + ($tee->handles)[0]->close ; is( undef, print( $tee "Argh1\n" ), 'teelaunch: write Argh1') ; is( undef, print( $tee "Argh2\n" ), 'teelaunch: write Argh2') ; # write not done @@ -18110,34 +18797,33 @@ sub tests_teelaunch is( 1, print( $tee "Argh 4\n" ), 'teelaunch: write Argh4 no') ; # Second time, lesson learnt IO::Tee - $mysync->{logfile} = 'W/tmp/tests/tests_teelaunch2.txt' ; - isa_ok( $tee = teelaunch( $mysync ), 'IO::Tee' , 'teelaunch: logfile W/tmp/tests/tests_teelaunch2.txt' ) ; + $tee = teelaunch( $mysync, 'W/tmp/tests/tests_teelaunch2.txt' ) ; + isa_ok( $tee, 'IO::Tee' , 'teelaunch: W/tmp/tests/tests_teelaunch2.txt' ) ; is( 1, print( $tee "Hi!\n" ), 'teelaunch: write Hi!') ; is( "Hi!\n", file_to_string( 'W/tmp/tests/tests_teelaunch2.txt' ), 'teelaunch: reading W/tmp/tests/tests_teelaunch2.txt is Hi!\n' ) ; is( 1, print( $tee "Hoo\n" ), 'teelaunch: write Hoo') ; is( "Hi!\nHoo\n", file_to_string( 'W/tmp/tests/tests_teelaunch2.txt' ), 'teelaunch: reading W/tmp/tests/tests_teelaunch2.txt is Hi!\nHoo\n' ) ; - is( 1, teefinish( $mysync ), 'teefinish: return 1') ; + is( 1, teefinish( $tee ), 'teefinish: return 1') ; is( 1, print( $tee "Argh1\n" ), 'teelaunch: write Argh1') ; is( 1, print( $tee "Argh2\n" ), 'teelaunch: write Argh2') ; is( "Hi!\nHoo\n", file_to_string( 'W/tmp/tests/tests_teelaunch2.txt' ), 'teelaunch: reading W/tmp/tests/tests_teelaunch2.txt is still Hi!\nHoo\n' ) ; - is( 1, teefinish( $mysync ), 'teefinish: still return 1') ; + is( 1, teefinish( $tee ), 'teefinish: still return 1') ; note( 'Leaving tests_teelaunch()' ) ; return ; } -sub teelaunch +sub teelaunch { - my $mysync = shift ; + my $mysync = shift @ARG ; + my $logfile = shift @ARG ; if ( ! defined( $mysync ) ) { return ; } - my $logfile = $mysync->{logfile} ; - if ( ! $logfile ) { return ; @@ -18145,38 +18831,41 @@ sub teelaunch logfileprepa( $logfile ) || croak "Error no valid directory to write log file $logfile : $OS_ERROR" ; + # To honor -T tainted mode. Could do better... + ( $logfile ) = $logfile =~ m{(.*)}x ; # This is a log file opened during the whole sync ## no critic (InputOutput::RequireBriefOpen) - open my $logfile_handle, '>', $logfile - or croak( "Can not open $logfile for write: $OS_ERROR" ) ; - binmode $logfile_handle, ":encoding(UTF-8)" ; - my $tee = IO::Tee->new( $logfile_handle, \*STDOUT ) ; - $tee->autoflush( 1 ) ; - $mysync->{logfile_handle} = $logfile_handle ; - $mysync->{tee} = $tee ; - return $tee ; + if ( ! open my $logfile_handle, '>', $logfile ) + { + carp( "Can not open $logfile for write: $OS_ERROR" ) ; + return ; + } + else + { + binmode $logfile_handle, ":encoding(UTF-8)" ; + my $tee = IO::Tee->new( $logfile_handle, \*STDOUT ) ; + $tee->autoflush( 1 ) ; + return $tee ; + } } -sub teefinish +sub teefinish { - my $mysync = shift ; - - if ( ! defined( $mysync ) ) { return ; } - - my $tee = $mysync->{tee} ; + my $tee = shift @ARG ; if ( ! defined( $tee ) ) { return ; } if ( 2 == scalar $tee->handles ) { - shift @{*{$tee}}; + my $handle = shift @{*{$tee}} ; + $handle->close ; } else { # nothing } return scalar $tee->handles ; -} +} sub getpwuid_any_os @@ -18255,6 +18944,8 @@ sub unittestssuite testsdebug( $mysync ) ; testunitsession( $mysync ) ; + cleanup_mess_from_tests( ) ; + my @summary = $test_builder->summary() ; my @details = $test_builder->details() ; my $nb_tests_run = scalar( @summary ) ; @@ -18267,15 +18958,25 @@ sub unittestssuite "List of failed tests:\n", $tests_failed ) ; return $EXIT_TESTS_FAILED ; } - - cleanup_mess_from_tests( ) ; - return 0 ; } sub cleanup_mess_from_tests { undef @pipemess ; + undef @include ; + undef @exclude ; + undef @folderrec ; + undef @folderfirst ; + undef @folderlast ; + undef @h1_folders_all ; + undef %h1_folders_all ; + undef @h2_folders_all ; + undef %h2_folders_all ; + undef @h2_folders_from_1_wanted ; + undef %h2_folders_from_1_all ; + undef %requested_folder ; + return ; } @@ -18418,7 +19119,7 @@ sub gmail12 $mysync->{maxbytesafter} ||= 1_000_000_000 ; # In fact it is documented as half: 500_000_000 $mysync->{automap} = ( defined $mysync->{automap} ) ? $mysync->{automap} : 1 ; $mysync->{maxsleep} = ( defined $mysync->{maxsleep} ) ? $mysync->{maxsleep} : $MAX_SLEEP ; ; - $skipcrossduplicates = ( defined $skipcrossduplicates ) ? $skipcrossduplicates : 0 ; + $mysync->{ skipcrossduplicates } = ( defined $mysync->{ skipcrossduplicates } ) ? $mysync->{ skipcrossduplicates } : 0 ; $mysync->{ synclabels } = ( defined $mysync->{ synclabels } ) ? $mysync->{ synclabels } : 1 ; $mysync->{ resynclabels } = ( defined $mysync->{ resynclabels } ) ? $mysync->{ resynclabels } : 1 ; push @useheader, 'X-Gmail-Received', 'Message-Id' ; @@ -18438,7 +19139,7 @@ sub gmail1 $mysync->{maxbytesafter} ||= 3_000_000_000 ; # $mysync->{automap} = ( defined $mysync->{automap} ) ? $mysync->{automap} : 1 ; $mysync->{maxsleep} = ( defined $mysync->{maxsleep} ) ? $mysync->{maxsleep} : $MAX_SLEEP ; ; - $skipcrossduplicates = ( defined $skipcrossduplicates ) ? $skipcrossduplicates : 1 ; + $mysync->{ skipcrossduplicates } = ( defined $mysync->{ skipcrossduplicates } ) ? $mysync->{ skipcrossduplicates } : 1 ; push @useheader, 'X-Gmail-Received', 'Message-Id' ; push @{ $mysync->{ regextrans2 } }, 's,\[Gmail\].,,' ; @@ -18454,9 +19155,7 @@ sub gmail2 $mysync->{ ssl2 } = ( defined $mysync->{ ssl2 } ) ? $mysync->{ ssl2 } : 1 ; $mysync->{ maxbytespersecond } ||= 20_000 ; # should be less than 10_000 computed from by Gmail documentation $mysync->{ maxbytesafter } ||= 1_000_000_000 ; # In fact it is documented as half: 500_000_000 - $mysync->{ automap } = ( defined $mysync->{ automap } ) ? $mysync->{ automap } : 1 ; - #$skipcrossduplicates = ( defined $skipcrossduplicates ) ? $skipcrossduplicates : 1 ; $mysync->{ expunge1 } = ( defined $mysync->{ expunge1 } ) ? $mysync->{ expunge1 } : 1 ; $mysync->{ addheader } = ( defined $mysync->{ addheader } ) ? $mysync->{ addheader } : 1 ; $mysync->{ maxsleep } = ( defined $mysync->{ maxsleep } ) ? $mysync->{ maxsleep } : $MAX_SLEEP ; ; @@ -18966,20 +19665,24 @@ sub tests_split_around_equal note( 'Entering tests_split_around_equal()' ) ; is( undef, split_around_equal( ), 'split_around_equal: no args => undef' ) ; - is_deeply( { toto => 'titi' }, { split_around_equal( 'toto=titi' ) }, 'split_around_equal: toto=titi => toto => titi' ) ; - is_deeply( { A => 'B', C => 'D' }, { split_around_equal( 'A=B=C=D' ) }, 'split_around_equal: toto=titi => toto => titi' ) ; + is_deeply( { toto => 'titi' }, { split_around_equal( 'toto=titi' ) }, 'split_around_equal: toto=titi => toto => titi' ) ; + is_deeply( { toto => undef }, { split_around_equal( 'toto' ) }, 'split_around_equal: tototiti => toto => undef' ) ; + is_deeply( { toto => '' }, { split_around_equal( 'toto=' ) }, 'split_around_equal: tototiti => toto= => empty' ) ; is_deeply( { A => 'B', C => 'D' }, { split_around_equal( 'A=B', 'C=D' ) }, 'split_around_equal: A=B C=D => A => B, C=>D' ) ; + is_deeply( { A => 'B', C => 'D', E => 'F' }, { split_around_equal( 'A=B', 'C=D', 'E=F' ) }, 'split_around_equal: A=B C=D => A => B, C=>D' ) ; + is_deeply( { A => 'B=C' }, { split_around_equal( 'A=B=C' ) }, 'split_around_equal: A=B=C => A => B=C' ) ; + is_deeply( { A => 'B=C=D' }, { split_around_equal( 'A=B=C=D' ) }, 'split_around_equal: A=B=C=D => A => B=C=D' ) ; note( 'Leaving tests_split_around_equal()' ) ; return ; } -sub split_around_equal -{ +sub split_around_equal +{ if ( ! @ARG ) { return ; } ; - return map { split /=/mxs, $_ } @ARG ; + return map { split( /=/mxs, $_, 2 ) } @ARG ; -} +} @@ -19526,7 +20229,7 @@ sub tests_get_options_from_string 'get_options_from_string: --nodebugimap1 --debugflags --errorsmax 2 => ok, empty string return' ) ; is( 0, $mysync->{ acc1 }->{ debugimap }, 'get_options_from_string: acc1->debugimap = 0 now' ) ; - is( 1, $debugflags, 'get_options_from_string: debugflags = 1 now' ) ; + is( 1, $mysync->{ debugflags }, 'get_options_from_string: debugflags = 1 now' ) ; is( 2, $mysync->{ errorsmax }, 'get_options_from_string: mysync->errorsmax = 2 now' ) ; is( '', get_options_from_string( $mysync, '--folder "IN BOX" --folder JOE' ), @@ -19552,7 +20255,7 @@ sub get_options_from_string 'debugimap!' => \$mysync->{ debugimap }, 'debugimap1!' => \$mysync->{ acc1 }->{ debugimap }, 'debugimap2!' => \$mysync->{ acc2 }->{ debugimap }, - 'debugflags!' => \$debugflags, + 'debugflags!' => \$mysync->{ debugflags }, 'debugsleep=f' => \$mysync->{ debugsleep }, 'errorsmax=i' => \$mysync->{ errorsmax }, 'folder=s@' => \$mysync->{ folder }, @@ -19688,7 +20391,7 @@ sub tests_get_options_cgi_context -sub get_options_cgi +sub get_options_cgi { # In CGI context arguments are not in @ARGV but in QUERY_STRING variable (with GET). my $mysync = shift @ARG ; @@ -19746,13 +20449,18 @@ sub get_options_cgi 'delete1!' => \$mysync->{ delete1 }, 'delete2!' => \$mysync->{ delete2 }, 'delete2duplicates!' => \$mysync->{ delete2duplicates }, - 'tail!' => \$mysync->{ tail }, - 'tmphash=s' => \$mysync->{ tmphash }, - 'exitwhenover=i' => \$mysync->{ exitwhenover }, - 'syncduplicates!' => \$mysync->{ syncduplicates }, - 'log!' => \$mysync->{ log }, - 'loglogfile!' => \$mysync->{ loglogfile }, - + 'tail!' => \$mysync->{ tail }, + 'tmphash=s' => \$mysync->{ tmphash }, + 'exitwhenover=i' => \$mysync->{ exitwhenover }, + 'syncduplicates!' => \$mysync->{ syncduplicates }, + 'skipcrossduplicates!' => \$mysync->{ skipcrossduplicates }, + 'debugcrossduplicates!'=> \$mysync->{ debugcrossduplicates }, + 'log!' => \$mysync->{ log }, + 'loglogfile!' => \$mysync->{ loglogfile }, + 'emailreportfrom=s' => \$mysync->{ email_report_from }, + 'emailreport1!' => \$mysync->{ emailreport1 }, + 'emailreport2!' => \$mysync->{ emailreport2 }, + 'var=s@' => \$mysync->{ var }, # f1f2h=s% could be removed but # tests_get_options_cgi() should be split before @@ -19765,9 +20473,9 @@ sub get_options_cgi return ; } return $numopt ; -} +} -sub get_options_cmd +sub get_options_cmd { my $mysync = shift @ARG ; my @arguments = @ARG ; @@ -19785,10 +20493,10 @@ sub get_options_cmd $mysync, \@arguments, 'debug!' => \$mysync->{ debug }, - 'debuglist!' => \$debuglist, + 'debuglist!' => \$mysync->{ debuglist }, 'debugcontent!' => \$mysync->{ debugcontent }, 'debugsleep=f' => \$mysync->{ debugsleep }, - 'debugflags!' => \$debugflags, + 'debugflags!' => \$mysync->{ debugflags }, 'debugimap!' => \$mysync->{ debugimap }, 'debugimap1!' => \$mysync->{ acc1 }->{ debugimap }, 'debugimap2!' => \$mysync->{ acc2 }->{ debugimap }, @@ -19958,12 +20666,12 @@ sub get_options_cmd 'sigignore=s@' => \$mysync->{ sigignore }, 'releasecheck!' => \$mysync->{releasecheck}, 'modulesversion|modules_version!' => \$modulesversion, - 'usecache!' => \$usecache, + 'usecache!' => \$mysync->{ usecache }, 'cacheaftercopy!' => \$cacheaftercopy, - 'debugcache!' => \$debugcache, - 'useuid!' => \$useuid, - 'addheader!' => \$mysync->{addheader}, - 'exitwhenover=i' => \$mysync->{ exitwhenover }, + 'debugcache!' => \$debugcache, + 'useuid!' => \$useuid, + 'addheader!' => \$mysync->{addheader}, + 'exitwhenover=i' => \$mysync->{ exitwhenover }, 'checkselectable!' => \$mysync->{ checkselectable }, 'checkfoldersexist!' => \$mysync->{ checkfoldersexist }, 'checkmessageexists!' => \$checkmessageexists, @@ -19982,8 +20690,9 @@ sub get_options_cmd 'maxbytespersecond=i' => \$mysync->{maxbytespersecond}, 'maxbytesafter=i' => \$mysync->{maxbytesafter}, 'maxsleep=f' => \$mysync->{maxsleep}, - 'skipcrossduplicates!' => \$skipcrossduplicates, - 'debugcrossduplicates!' => \$debugcrossduplicates, + 'syncduplicates!' => \$mysync->{ syncduplicates }, + 'skipcrossduplicates!' => \$mysync->{ skipcrossduplicates }, + 'debugcrossduplicates!' => \$mysync->{ debugcrossduplicates }, 'log!' => \$mysync->{log}, 'tail!' => \$mysync->{tail}, 'logfile=s' => \$mysync->{logfile}, @@ -20000,10 +20709,11 @@ sub get_options_cmd 'justfolderlists!' => \$mysync->{justfolderlists}, 'delete1emptyfolders' => \$mysync->{delete1emptyfolders}, 'checknoabletosearch!' => \$mysync->{checknoabletosearch}, - 'syncduplicates!' => \$mysync->{ syncduplicates }, 'dockercontext!' => \$mysync->{ dockercontext }, - - + 'emailreportfrom=s' => \$mysync->{ email_report_from }, + 'emailreport1!' => \$mysync->{ emailreport1 }, + 'emailreport2!' => \$mysync->{ emailreport2 }, + 'var=s@' => \$mysync->{ var }, ) ; #myprint( Data::Dumper->Dump( [ $mysync ] ) ) ; $mysync->{ debug } and output( $mysync, "get options: [$opt_ret][$numopt]\n" ) ; @@ -20107,7 +20817,7 @@ sub get_options } -sub tests_infos +sub tests_infos { note( 'Entering tests_infos()' ) ; note( "OSNAME=$OSNAME" ) ; @@ -20125,7 +20835,6 @@ sub tests_infos cpu_number note( "cpu_number: " . cpu_number() ) ; note( $sync->{rcs} ) ; - note( 'Leaving tests_infos()' ) ; return ; } @@ -20272,7 +20981,7 @@ sub testsunit return ; } -sub testsdebug +sub testsdebug { # Now a little obsolete since there is # imapsync ... --testsunit "anyfunction" @@ -20293,14 +21002,17 @@ sub testsdebug #tests_probe_imapssl( ) ; #tests_cpu_number( ) ; #tests_mailimapclient_connect( ) ; - tests_loadavg( ) ; + #tests_loadavg( ) ; #tests_always_fail( ) ; - + tests_localtimez( ) ; + tests_year_month_day_hour_min_sec_ms( ) ; + tests_date_rfc822( ) ; + tests_email_report_message_id( ) ; note( 'Leaving testsdebug()' ) ; done_testing( ) ; } return ; -} +} sub tests @@ -20466,7 +21178,7 @@ sub tests tests_eta( ) ; tests_timesince( ) ; tests_timenext( ) ; - tests_foldersize( ) ; + tests_imapsync_context( ) ; tests_abort( ) ; tests_probe_imapssl( ) ; @@ -20499,6 +21211,18 @@ sub tests tests_compress( ) ; tests_get_options_extra( ) ; tests_get_options_from_string( ) ; + tests_email_report_message_id( ) ; + tests_year_month_day_hour_min_sec_ms( ) ; + tests_fractional_of_floor( ) ; + tests_date_rfc822( ) ; + tests_email_report_from( ) ; + tests_email_report_to( ) ; + tests_email_report_body_base( ) ; + tests_email_report( ) ; + tests_setlogdir( ) ; + tests_logfilesuffix( ) ; + tests_cgienvcontext( ) ; + tests_usecache_and_skipcrossduplicates( ) ; tests_infos( ) ; #tests_resolv( ) ; @@ -20509,11 +21233,11 @@ sub tests #tests_kill_zero( ) ; #tests_always_fail( ) ; - done_testing( 1860 ) ; + done_testing( 1922 ) ; note( 'Leaving tests()' ) ; } return ; -} +} sub tests_template { @@ -20536,4 +21260,3 @@ sub template return ; } - diff --git a/index.shtml b/index.shtml index f6f0600..a9a327e 100644 --- a/index.shtml +++ b/index.shtml @@ -74,7 +74,8 @@

    Afraid of losing all your precious emails?
    It's time to copy all of them elsewhere!
    -Safely. +Safely.
    +And welcome to ex-Free Google Workspace people!

    @@ -880,6 +881,7 @@ you some struggling hours later, believe me. So read the three columns main
  • IceWarp.
  • DBmail.
  • FirstClass.
    • +
  • James.
  • Other various imap software servers.
@@ -987,7 +989,7 @@ like written in the https://imapsync.lamiral.info/LICENSE This document last modified on -($Id: index.shtml,v 1.508 2021/12/20 17:49:51 gilles Exp gilles $)
+($Id: index.shtml,v 1.511 2022/04/06 10:38:35 gilles Exp gilles $)
Top of the page

diff --git a/tests.sh b/tests.sh index 3503501..3c7d431 100644 --- a/tests.sh +++ b/tests.sh @@ -1,6 +1,6 @@ #!/bin/sh -# $Id: tests.sh,v 1.367 2022/01/13 12:59:05 gilles Exp gilles $ +# $Id: tests.sh,v 1.372 2022/04/06 10:07:32 gilles Exp gilles $ # To run these tests, you need a running imap server somewhere # with several accounts. And be on Linux or Unix. @@ -239,14 +239,14 @@ option_tests_in_var_tmp_sub() { ( mkdir -p /var/tmp/imapsync_tests cd /var/tmp/imapsync_tests - /g/public_html/imapsync/i3 --tests + /g/public_html/imapsync/imapsync --tests ) } option_tests_in_var_tmp() { ( cd /var/tmp/ - /g/public_html/imapsync/i3 --tests + /g/public_html/imapsync/imapsync --tests ) } @@ -420,6 +420,21 @@ ll() { --passfile2 ../../var/pass/secret.titi } +ll_diff_log_stdout_debugssl() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --justlogin --debugssl 4 \ + --logfile ll_diff_log_stdout_debugssl_1.txt 2>&1 \ + | tee LOG_imapsync/ll_diff_log_stdout_debugssl_2.txt + echo + diff LOG_imapsync/ll_diff_log_stdout_debugssl_1.txt LOG_imapsync/ll_diff_log_stdout_debugssl_2.txt +} + + + ll_INBOX() { $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 tata \ @@ -552,6 +567,17 @@ ll_search_larger() { --search 'LARGER 1000' --folder INBOX } +ll_search_keyword() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --search 'KEYWORD NonJunk' --folder INBOX.flagsetSeen --debugflags --debugimap1 +} + + + ll_maxsize() { $CMD_PERL ./imapsync \ @@ -1087,6 +1113,19 @@ pidfile_bad() { test "$?" = "$EXIT_PID_FILE_ERROR" } + +ll_skipcrossduplicates_usecache() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST2 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --skipcrossduplicates --usecache + test "$?" = "$EX_USAGE" +} + + + test_tail() { $CMD_PERL ./imapsync \ --justbanner --simulong 15 \ @@ -3145,7 +3184,7 @@ ll_f1f2_01() --passfile2 ../../var/pass/secret.titi \ --justfolders \ --folder 'INBOX.yop.yap' --f1f2 'INBOX.yop.yap=INBOX/rha/lovely' --f1f2 'lalala=lululu' --debugfolders - + test "$EXIT_ERR_CREATE" = "$?" } ll_regextrans2() @@ -4855,6 +4894,8 @@ ll_duplicates_across_folders() { --skipcrossduplicates --debugcrossduplicates } + + ll_delete2_dev() { can_send && sendtestmessage titi can_send && sendtestmessage @@ -4995,6 +5036,26 @@ ll_syncduplicates() { --folder INBOX.duplicates --debug --syncduplicates # --dry } +ll_syncduplicates_delete2() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST1 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.duplicates --syncduplicates --delete2 # --dry +} + +ll_syncduplicates_delete2_delete2duplicates() { + $CMD_PERL ./imapsync \ + --host1 $HOST1 --user1 tata \ + --passfile1 ../../var/pass/secret.tata \ + --host2 $HOST1 --user2 titi \ + --passfile2 ../../var/pass/secret.titi \ + --folder INBOX.duplicates --syncduplicates --delete2 --delete2duplicates # --dry +} + + + ll_syncduplicates_noskipsize() { $CMD_PERL ./imapsync \ --host1 $HOST1 --user1 tata \ @@ -5858,6 +5919,19 @@ gmail_gmail_9_search_X_GM_LABELS() { --search 'X-GM-LABELS "Important"' } +gmail_gmail_10_search_drafts() { + ! ping -c1 imap.gmail.com || $CMD_PERL ./imapsync \ + --gmail1 \ + --user1 gilles.lamiral@gmail.com \ + --passfile1 ../../var/pass/secret.gilles_gmail \ + --gmail2 \ + --user2 imapsync.gl@gmail.com \ + --passfile2 ../../var/pass/secret.imapsync.gl_gmail \ + --nofoldersizes \ + --folderfirst '[Gmail]/Drafts' --debuglabels --dry \ + --folder Test --folder '[Gmail]/Drafts' +} + @@ -5911,7 +5985,7 @@ gmail_gl1_gl2_labels() --user2 imapsync.gl2@gmail.com \ --passfile2 ../../var/pass/secret.imapsync.gl2_gmail \ --exclude "\[Gmail\]" \ - --synclabels --resynclabels --debuglabels --dry + --synclabels --resynclabels --debug --debuglabels # --dry } gmail_gl1_gl2_labels_subfolder2() @@ -6921,7 +6995,7 @@ huge_folder() --host2 $HOST2 --user2 tete@est.belle \ --passfile2 ../../var/pass/secret.tete \ --include INBOX.Junk.2010 \ - --nousecache --tmpdir /var/tmp --debugmemory || \ + --tmpdir /var/tmp --debugmemory || \ true } date2=`date` @@ -7263,31 +7337,38 @@ dprof2_bigmail() curl_online_args() { - curl -v -d 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;simulong=2' \ + curl -v --data 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;simulong=2' \ https://imapsync.lamiral.info/cgi-bin/imapsync } +curl_online_testslive() +{ + curl -v --data 'testslive=1;simulong=2' https://imapsync.lamiral.info/cgi-bin/imapsync +} + + + curl_online_args_pidfile() { - curl -v -d "host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;pidfile=/tmp/curl_online_args_pidfile_$$.txt" \ + curl -v --data"host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;pidfile=/tmp/curl_online_args_pidfile_$$.txt" \ https://imapsync.lamiral.info/cgi-bin/imapsync } curl_online_args_nolog() { - curl -v -d 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;justbanner=1;log=' \ + curl -v --data 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;justbanner=1;log=' \ https://lamiral.info/cgi-bin/imapsync } curl_online_args_nolog_2() { - curl -v -d 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;justbanner=1;log=0' \ + curl -v --data 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;justbanner=1;log=0' \ https://lamiral.info/cgi-bin/imapsync } curl_online_justbanner() { - curl -v -d 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;simulong=0.7;justbanner=1' \ + curl -v --data 'host1=test1.lamiral.info;user1=test1;password1=secret1;host2=test2.lamiral.info;user2=test2;password2=secret2;simulong=0.7;justbanner=1' \ https://lamiral.info/cgi-bin/imapsync } @@ -7305,14 +7386,14 @@ simulong=2; dry=1; EOF - curl -v -d '@W/tmp/cred.txt' \ + curl -v --data '@W/tmp/cred.txt' \ https://imapsync.lamiral.info/cgi-bin/imapsync } curl_online_args_json() { # DO NOT WORK AT ALL - ! curl -v -d '{ "testslive":"1" }' -H "Content-Type: application/json" \ + ! curl -v --data '{ "testslive":"1" }' -H "Content-Type: application/json" \ https://imapsync.lamiral.info/cgi-bin/imapsync } @@ -7357,7 +7438,7 @@ curl_online_external() https://web-tools.na.icb.cnr.it/cgi-bin/imapsync \ https://140.164.23.4/cgi-bin/imapsync \ ; do - curl -k -s -d 'justconnect=1;host1=mail.unionstrategiesinc.com;user1=a;user2=a;host2=mail5.unionstrategiesinc.com;simulong=2' \ + curl -k -s --data 'justconnect=1;host1=mail.unionstrategiesinc.com;user1=a;user2=a;host2=mail5.unionstrategiesinc.com;simulong=2' \ $imapsync #sleep 2 done @@ -7416,6 +7497,7 @@ ll_unknow_option ll_ask_password ll_env_password ll_bug_folder_name_with_blank +ll_skipcrossduplicates_usecache ll_timeout ll_timeout1_timeout2 ll_timeout_very_small @@ -7436,6 +7518,7 @@ ll_buffersize ll_justfolders ll_justfolders_delete1emptyfolders ll_justfolders_skipemptyfolders +ll_f1f2_01 ll_prefix12 ll_nosyncinternaldates ll_idatefromheader @@ -7558,6 +7641,7 @@ ll_abort_byfile_hand_made ll_abort_byfile_imapsync_made ll_abort_byfile_normal_run ll_sigreconnect_INT +ll_diff_log_stdout_debugssl curl_online_args curl_online_file ksks_reset_test1 @@ -7590,7 +7674,7 @@ set_return_code_variables if test $# -eq 0; then # mandatory tests if run_tests $mandatory_tests; then - ./i3 --version >> .test_3xx + ./imapsync --version >> .tests_passed return 0 fi else