From ee992278e818135fc5499ef84a54d08e2807c7b6 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 10 May 2020 21:56:05 -0600 Subject: [PATCH] WIP on QWK download support --- core/enig_error.js | 1 + core/file_base_user_list_export.js | 3 +- core/message_base_qwk_export.js | 378 +++++++++++++++++++++++++++++ core/qwk_mail_packet.js | 7 +- 4 files changed, 387 insertions(+), 2 deletions(-) create mode 100644 core/message_base_qwk_export.js diff --git a/core/enig_error.js b/core/enig_error.js index 08a3312e..be025214 100644 --- a/core/enig_error.js +++ b/core/enig_error.js @@ -37,6 +37,7 @@ exports.Errors = { MissingParam : (reason, reasonCode) => new EnigError('Missing paramter(s)', -32008, reason, reasonCode), MissingMci : (reason, reasonCode) => new EnigError('Missing required MCI code(s)', -32009, reason, reasonCode), BadLogin : (reason, reasonCode) => new EnigError('Bad login attempt', -32010, reason, reasonCode), + UserInterrupt : (reason, reasonCode) => new EnigError('User interrupted', -32011, reason, reasonCode), }; exports.ErrorReasons = { diff --git a/core/file_base_user_list_export.js b/core/file_base_user_list_export.js index 3c00d167..1307144c 100644 --- a/core/file_base_user_list_export.js +++ b/core/file_base_user_list_export.js @@ -28,7 +28,7 @@ const yazl = require('yazl'); tsFormat - timestamp format (theme 'short') descWidth - max desc width (45) progBarChar - progress bar character (▒) - compressThreshold - threshold to kick in comrpession for lists (1.44 MiB) + compressThreshold - threshold to kick in compression for lists (1.44 MiB) templates - object containing: header - filename of header template (misc/file_list_header.asc) entry - filename of entry template (misc/file_list_entry.asc) @@ -244,6 +244,7 @@ exports.getModule = class FileBaseListExport extends MenuModule { }, function done(callback) { // re-enable idle monitor + // :TODO: this should probably be moved down below at the end of the full waterfall self.client.startIdleMonitor(); updateStatus('Exported list has been added to your download queue'); diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js new file mode 100644 index 00000000..b79a6411 --- /dev/null +++ b/core/message_base_qwk_export.js @@ -0,0 +1,378 @@ +// ENiGMA½ +const { MenuModule } = require('./menu_module'); +const Message = require('./message'); +const { Errors } = require('./enig_error'); +const { + getMessageAreaByTag, + hasMessageConfAndAreaRead, +} = require('./message_area'); +const FileArea = require('./file_base_area'); +const { QWKPacketWriter } = require('./qwk_mail_packet'); +const { renderSubstr } = require('./string_util'); +const Config = require('./config').get; +const FileEntry = require('./file_entry'); +const Events = require('./events'); +const DownloadQueue = require('./download_queue'); + +// deps +const async = require('async'); +const _ = require('lodash'); +const fse = require('fs-extra'); +const temptmp = require('temptmp'); +const paths = require('path'); +const UUIDv4 = require('uuid/v4'); + +const FormIds = { + main : 0, +}; + +const MciViewIds = { + main : { + status : 1, + progressBar : 2, + + customRangeStart : 10, + } +}; + +exports.moduleInfo = { + name : 'QWK Export', + desc : 'Exports a QWK Packet for download', + author : 'NuSkooler', +}; + +exports.getModule = class MessageBaseQWKExport extends MenuModule { + constructor(options) { + super(options); + + this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs); + + this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1); + this.config.bbsID = this.config.bbsID || _.get(Config(), 'messageNetworks.qwk.bbsID', 'ENIGMA'); + + this.tempName = `${UUIDv4().substr(-8).toUpperCase()}.QWK`; + this.sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads); + } + + mciReady(mciData, cb) { + super.mciReady(mciData, err => { + if (err) { + return cb(err); + } + + async.waterfall( + [ + (callback) => { + this.prepViewController('main', FormIds.main, mciData.menu, err => { + return callback(err); + }); + }, + (callback) => { + this.temptmp = temptmp.createTrackedSession('qwkuserexp'); + this.temptmp.mkdir({ prefix : 'enigqwkwriter-'}, (err, tempDir) => { + if (err) { + return callback(err); + } + + this.tempPacketDir = tempDir; + + const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(this.sysTempDownloadArea); + + // ensure dir exists + fse.mkdirs(sysTempDownloadDir, err => { + return callback(err, sysTempDownloadDir); + }); + }); + }, + (sysTempDownloadDir, callback) => { + this.performExport(sysTempDownloadDir, err => { + return callback(err); + }); + }, + ], + err => { + this.temptmp.cleanup(); + + if (err) { + // :TODO: doesn't do anything currently: + if ('NORESULTS' === err.reasonCode) { + return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'qwkExportNoResults'); + } + + return this.prevMenu(); + } + return cb(err); + } + ); + }); + } + + finishedLoading() { + this.prevMenu(); + } + + performExport(sysTempDownloadDir, cb) { + const statusView = this.viewControllers.main.getView(MciViewIds.main.status); + const updateStatus = (status) => { + if (statusView) { + statusView.setText(status); + } + }; + + const progBarView = this.viewControllers.main.getView(MciViewIds.main.progressBar); + const updateProgressBar = (curr, total) => { + if (progBarView) { + const prog = Math.floor( (curr / total) * progBarView.dimens.width ); + progBarView.setText(this.config.progBarChar.repeat(prog)); + } + }; + + let cancel = false; + + let lastProgUpdate = 0; + const progressHandler = (state, next) => { + // we can produce a TON of updates; only update progress at most every 3/4s + if (Date.now() - lastProgUpdate > 750) { + switch (state.step) { + case 'next_area' : + updateStatus(state.status); + updateProgressBar(0, 0); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.area); + break; + + case 'message' : + updateStatus(state.status); + updateProgressBar(state.current, state.total); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.message); + break; + + default : + break; + } + lastProgUpdate = Date.now(); + } + + return next(cancel ? Errors.UserInterrupt('User canceled') : null); + }; + + const keyPressHandler = (ch, key) => { + if('escape' === key.name) { + cancel = true; + this.client.removeListener('key press', keyPressHandler); + } + }; + + const processMessagesWithFilter = (filter, cb) => { + Message.findMessages(filter, (err, messageIds) => { + if (err) { + return cb(err); + } + + let current = 1; + async.eachSeries(messageIds, (messageId, nextMessageId) => { + const message = new Message(); + message.load({ messageId }, err => { + if (err) { + return nextMessageId(err); + } + + const progress = { + current, + message, + step : 'message', + total : messageIds.length, + status : `Writing message ${current} / ${messageIds.length}`, + }; + + progressHandler(progress, err => { + if (err) { + return nextMessageId(err); + } + + packetWriter.appendMessage(message); + current += 1; + + return nextMessageId(null); + }); + }); + }, + err => { + return cb(err); + }); + }); + }; + + const packetWriter = new QWKPacketWriter({ + user : this.client.user, + bbsID : this.config.bbsID, + }); // :TODO: User configuration here + + packetWriter.on('warning', warning => { + this.client.log.warn( { warning }, 'QWK packet writer warning'); + }); + + async.waterfall( + [ + (callback) => { + // don't count idle monitor while processing + this.client.stopIdleMonitor(); + + // let user cancel + this.client.on('key press', keyPressHandler); + + packetWriter.once('ready', () => { + return callback(null); + }); + + packetWriter.once('error', err => { + this.client.log.error( { error : err.message }, 'QWK packet writer error'); + cancel = true; + }); + + packetWriter.init(); + }, + (callback) => { + // + // Fetch messages for user-configured area tags. + // - If private tag is present, we fetch this separately. + // - User property determines newscan timestamps dates if present for tag, else "all" + // - We have to fetch one area at a time in order to process message pointers/timestamps. + // ...this also allows for better progress. + // + // TL;DR: for each area -> for each message + // + const exportAreas = [ // :TODO: Load in something like this + { + areaTag : 'general', + newerThanTimestamp : '2018-01-01', + }, + { + areaTag : 'fsx_gen', + } + ]; + + async.eachSeries(exportAreas, (exportArea, nextExportArea) => { + const area = getMessageAreaByTag(exportArea.areaTag); + if (!area) { + // :TODO: remove from user properties - this area does not exist + this.client.log.warn({ areaTag : exportArea.areaTag }, 'Cannot QWK export area as it does not exist'); + return nextExportArea(null); + } + + if (!hasMessageConfAndAreaRead(this.client, area)) { + this.client.log.warn({ areaTag : area.areaTag }, 'Cannot QWK export area due to ACS'); + return nextExportArea(null); + } + + const progress = { + area, + step : 'next_area', + status : `Gathering messages in ${area.name}...`, + }; + + progressHandler(progress, err => { + if (err) { + return nextExportArea(err); + } + + const filter = { + resultType : 'id', + areaTag : exportArea.areaTag, + newerThanTimestamp : exportArea.newerThanTimestamp + }; + + processMessagesWithFilter(filter, err => { + return nextExportArea(err); + }); + }); + }, + err => { + return callback(err); + }); + }, + (callback) => { + const filter = { + resultType : 'id', + privateTagUserId : this.client.user.userId, + // :TODO: newerThanTimestamp for private messages + //newerThanTimestamp : exportArea.newerThanTimestamp + }; + return processMessagesWithFilter(filter, callback); + }, + (callback) => { + let packetInfo; + packetWriter.once('packet', info => { + packetInfo = info; + }); + + packetWriter.once('finished', () => { + return callback(null, packetInfo); + }); + + packetWriter.finish(this.tempPacketDir); + }, + (packetInfo, callback) => { + const sysDownloadPath = paths.join(sysTempDownloadDir, this.tempName); + fse.move(packetInfo.path, sysDownloadPath, err => { + return callback(null, sysDownloadPath, packetInfo); + }); + }, + (sysDownloadPath, packetInfo, callback) => { + const newEntry = new FileEntry({ + areaTag : this.sysTempDownloadArea.areaTag, + fileName : paths.basename(sysDownloadPath), + storageTag : this.sysTempDownloadArea.storageTags[0], + meta : { + upload_by_username : this.client.user.username, + upload_by_user_id : this.client.user.userId, + byte_size : packetInfo.stats.size, + session_temp_dl : 1, // download is valid until session is over + + // :TODO: something like this: allow to override the displayed/downloaded as filename + // separate from the actual on disk filename. E.g. we could always download as "ENIGMA.QWK" + visible_filename : paths.basename(packetInfo.path), + } + }); + + newEntry.desc = 'QWK Export'; + + newEntry.persist(err => { + if(!err) { + // queue it! + const dlQueue = new DownloadQueue(this.client); + dlQueue.add(newEntry, true); // true=systemFile + + // clean up after ourselves when the session ends + // :TODO: DRY this with that in file_base_user_export + const thisClientId = this.client.session.id; + Events.once(Events.getSystemEvents().ClientDisconnected, evt => { + if(thisClientId === _.get(evt, 'client.session.id')) { + FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { + if(err) { + Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); + } else { + Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); + } + }); + } + }); + } + return callback(err); + }); + } + ], + err => { + this.client.startIdleMonitor(); // re-enable + this.client.removeListener('key press', keyPressHandler); + + if (!err) { + updateStatus('A QWK packet has been placed in your download queue'); + } + + // :TODO: send user to download manager with pop flags/etc. + + return cb(err); + } + ); + } +}; \ No newline at end of file diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index eec737cb..3d613ca4 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1242,7 +1242,12 @@ class QWKPacketWriter extends EventEmitter { files, this.workDir, err => { - return cb(err); + fs.stat(packetPath, (err, stats) => { + if (stats) { + this.emit('packet', { stats, path : packetPath } ); + } + return cb(err); + }); } ); });