diff --git a/core/database.js b/core/database.js index 35e9cc87..3baf682a 100644 --- a/core/database.js +++ b/core/database.js @@ -203,17 +203,6 @@ function createMessageBaseTables() { ' UNIQUE(user_id, area_tag)' + ');' ); - - // :TODO: Not currently used - dbs.message.run( - 'CREATE TABLE IF NOT EXISTS user_message_status (' + - ' user_id INTEGER NOT NULL,' + - ' message_id INTEGER NOT NULL,' + - ' status INTEGER NOT NULL,' + - ' UNIQUE(user_id, message_id, status),' + - ' FOREIGN KEY(user_id) REFERENCES user(id)' + - ');' - ); dbs.message.run( `CREATE TABLE IF NOT EXISTS message_area_last_scan ( diff --git a/core/ftn_util.js b/core/ftn_util.js index fdc5e5a6..e46ee12c 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -68,7 +68,8 @@ function getDateFromFtnDateTime(dateTime) { // "27 Feb 15 00:00:03" // // :TODO: Use moment.js here - return (new Date(Date.parse(dateTime))).toISOString(); + return moment(Date.parse(dateTime)); // Date.parse() allows funky formats +// return (new Date(Date.parse(dateTime))).toISOString(); } function getDateTimeString(m) { diff --git a/core/message.js b/core/message.js index 30b72d7c..a9f74b0f 100644 --- a/core/message.js +++ b/core/message.js @@ -1,14 +1,15 @@ /* jslint node: true */ 'use strict'; -var msgDb = require('./database.js').dbs.message; -var wordWrapText = require('./word_wrap.js').wordWrapText; -var ftnUtil = require('./ftn_util.js'); +let msgDb = require('./database.js').dbs.message; +let wordWrapText = require('./word_wrap.js').wordWrapText; +let ftnUtil = require('./ftn_util.js'); -var uuid = require('node-uuid'); -var async = require('async'); -var _ = require('lodash'); -var assert = require('assert'); +let uuid = require('node-uuid'); +let async = require('async'); +let _ = require('lodash'); +let assert = require('assert'); +let moment = require('moment'); module.exports = Message; @@ -24,10 +25,10 @@ function Message(options) { this.subject = options.subject || ''; this.message = options.message || ''; - if(_.isDate(options.modTimestamp)) { - this.modTimestamp = options.modTimestamp; + if(_.isDate(options.modTimestamp) || moment.isMoment(options.modTimestamp)) { + this.modTimestamp = moment(options.modTimestamp); } else if(_.isString(options.modTimestamp)) { - this.modTimestamp = new Date(options.modTimestamp); + this.modTimestamp = moment(options.modTimestamp); } this.viewCount = options.viewCount || 0; @@ -44,11 +45,8 @@ function Message(options) { this.meta = options.meta; } -// this.meta = options.meta || {}; this.hashTags = options.hashTags || []; - var self = this; - this.isValid = function() { // :TODO: validate as much as possible return true; @@ -59,8 +57,8 @@ function Message(options) { }; this.getMessageTimestampString = function(ts) { - ts = ts || new Date(); - return ts.toISOString(); + ts = ts || moment(); + return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ'); }; } @@ -70,13 +68,7 @@ Message.WellKnownAreaTags = { Bulletin : 'local_bulletin', }; -// :TODO: This doesn't seem like a good way to go -- perhaps only for local/user2user, or just use -// a system similar to the "last read" for general areas -Message.Status = { - New : 0, - Read : 1, -}; - +// :TODO: FTN stuff really doesn't belong here - move it elsewhere and/or just use the names directly when needed Message.MetaCategories = { System : 1, // ENiGMA1/2 stuff FtnProperty : 2, // Various FTN network properties, ftn_cost, ftn_origin, ... @@ -86,6 +78,13 @@ Message.MetaCategories = { Message.SystemMetaNames = { LocalToUserID : 'local_to_user_id', LocalFromUserID : 'local_from_user_id', + StateFlags0 : 'state_flags0', // See Message.StateFlags0 +}; + +Message.StateFlags0 = { + None : 0x00000000, + Imported : 0x00000001, // imported from foreign system + Exported : 0x00000002, // exported to foreign system }; Message.FtnPropertyNames = { @@ -152,7 +151,31 @@ Message.getMessageIdsByMetaValue = function(category, name, value, cb) { ); }; -Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) { +Message.getMetaValuesByMessageId = function(messageId, category, name, cb) { + const sql = + `SELECT meta_value + FROM message_meta + WHERE message_id = ? AND meta_category = ? AND meta_name = ?;`; + + msgDb.all(sql, [ messageId, category, name ], (err, rows) => { + if(err) { + return cb(err); + } + + if(0 === rows.length) { + return cb(new Error('No value for category/name')); + } + + // single values are returned without an array + if(1 === rows.length) { + return cb(null, rows[0].meta_value); + } + + cb(null, rows.map(r => r.meta_value)); // map to array of values only + }); +}; + +Message.getMetaValuesByMessageUuid = function(uuid, category, name, cb) { async.waterfall( [ function getMessageId(callback) { @@ -160,32 +183,14 @@ Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, callback(err, messageId); }); }, - function getMetaValue(messageId, callback) { - const sql = - `SELECT meta_value - FROM message_meta - WHERE message_id = ? AND message_category = ? AND meta_name = ?;`; - - msgDb.all(sql, [ messageId, category, name ], (err, rows) => { - if(err) { - return callback(err); - } - - if(0 === rows.length) { - return callback(new Error('No value for category/name')); - } - - // single values are returned without an array - if(1 === rows.length) { - return callback(null, rows[0].meta_value); - } - - callback(null, rows.map(r => r.meta_value)); + function getMetaValues(messageId, callback) { + Message.getMetaValuesByMessageId(messageId, category, name, (err, values) => { + callback(err, values); }); } ], - (err, value) => { - cb(err, value); + (err, values) => { + cb(err, values); } ); }; @@ -254,7 +259,7 @@ Message.prototype.load = function(options, cb) { self.fromUserName = msgRow.from_user_name; self.subject = msgRow.subject; self.message = msgRow.message; - self.modTimestamp = msgRow.modified_timestamp; + self.modTimestamp = moment(msgRow.modified_timestamp); self.viewCount = msgRow.view_count; callback(err); @@ -269,12 +274,6 @@ Message.prototype.load = function(options, cb) { function loadHashTags(callback) { // :TODO: callback(null); - }, - function loadMessageStatus(callback) { - if(options.user) { - // :TODO: Load from user_message_status - } - callback(null); } ], function complete(err) { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index b112d474..fb5d22fe 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -29,13 +29,12 @@ exports.moduleInfo = { }; /* - :TODO: - * Add bundle timer (arcmail) - * Queue until time elapses / fixed time interval - * Pakcets append until >= max byte size - * [if arch type is not empty): Packets -> bundle until max byte size -> repeat process - * NetMail needs explicit isNetMail() check - * NetMail filename / location / etc. is still unknown - need to post on groups & get real answers + :TODO: + * Support (approx) max bundle size + * Support NetMail + * NetMail needs explicit isNetMail() check + * NetMail filename / location / etc. is still unknown - need to post on groups & get real answers + */ exports.getModule = FTNMessageScanTossModule; @@ -242,9 +241,7 @@ function FTNMessageScanTossModule() { message.meta.FtnProperty.ftn_dest_node = options.destAddress.node; message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net; message.meta.FtnProperty.ftn_dest_network = options.destAddress.net; - // :TODO: attr1 & 2 message.meta.FtnProperty.ftn_cost = 0; - message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); // :TODO: Need an explicit isNetMail() check @@ -308,15 +305,15 @@ function FTNMessageScanTossModule() { } message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); - - if(!message.meta.FtnKludge.PID) { - message.meta.FtnKludge.PID = ftnUtil.getProductIdentifier(); - } - - if(!message.meta.FtnKludge.TID) { - // :TODO: Create TID!! - //message.meta.FtnKludge.TID = - } + + // + // According to FSC-0046: + // + // "When a Conference Mail processor adds a TID to a message, it may not + // add a PID. An existing TID should, however, be replaced. TIDs follow + // the same format used for PIDs, as explained above." + // + message.meta.FtnKludge.TID = ftnUtil.getProductIdentifier(); // // Determine CHRS and actual internal encoding name @@ -331,13 +328,32 @@ function FTNMessageScanTossModule() { options.encoding = encoding; // save for later message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding); - // :TODO: FLAGS kludge? - // :TODO: Add REPLY kludge if appropriate - + // :TODO: FLAGS kludge? }; + this.setReplyKludgeFromReplyToMsgId = function(message, cb) { + // + // Look up MSGID kludge for |message.replyToMsgId|, if any. + // If found, we can create a REPLY kludge with the previously + // discovered MSGID. + // + + if(0 === message.replyToMsgId) { + return cb(null); // nothing to do + } + + Message.getMetaValuesByMessageId(message.replyToMsgId, 'FtnKludge', 'MSGID', (err, msgIdVal) => { + assert(_.isString(msgIdVal)); + + if(!err) { + // got a MSGID - create a REPLY + message.meta.FtnKludge.REPLY = msgIdVal; + } + + cb(null); // this method always passes + }); + }; - // :TODO: change to something like isAreaConfigValid // check paths, Addresses, etc. this.isAreaConfigValid = function(areaConfig) { if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) { @@ -415,7 +431,7 @@ function FTNMessageScanTossModule() { }); }; - this.getNodeConfigKeyForUplink = function(uplink) { + this.getNodeConfigKeyByAddress = function(uplink) { // :TODO: sort by least # of '*' & take top? const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { return Address.fromString(addr).isPatternMatch(uplink); @@ -453,14 +469,20 @@ function FTNMessageScanTossModule() { }); } else { callback(null); - } + } }, function loadMessage(callback) { message.load( { uuid : msgUuid }, err => { - if(!err) { - self.prepareMessage(message, exportOpts); + if(err) { + return callback(err); } - callback(err); + + // General preperation + self.prepareMessage(message, exportOpts); + + self.setReplyKludgeFromReplyToMsgId(message, err => { + callback(err); + }); }); }, function createNewPacket(callback) { @@ -502,7 +524,12 @@ function FTNMessageScanTossModule() { } callback(null); }, - function updateStoredMeta(callback) { + function storeStateFlags0Meta(callback) { + message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => { + callback(err); + }); + }, + function storeMsgIdMeta(callback) { // // We want to store some meta as if we had imported // this message for later reference @@ -577,7 +604,7 @@ function FTNMessageScanTossModule() { this.exportMessagesToUplinks = function(messageUuids, areaConfig, cb) { async.each(areaConfig.uplinks, (uplink, nextUplink) => { - const nodeConfigKey = self.getNodeConfigKeyForUplink(uplink); + const nodeConfigKey = self.getNodeConfigKeyByAddress(uplink); if(!nodeConfigKey) { return nextUplink(); } @@ -657,7 +684,6 @@ function FTNMessageScanTossModule() { const newPath = paths.join(outgoingDir, paths.basename(oldPath)); fs.rename(oldPath, newPath, err => { if(err) { - // :TODO: Log this - but move on to the next file Log.warn( { oldPath : oldPath, newPath : newPath }, 'Failed moving temporary bundle file!'); @@ -756,6 +782,10 @@ function FTNMessageScanTossModule() { }); }, function persistImport(callback) { + // mark as imported + message.meta.System.StateFlags0 = Message.StateFlags0.Imported.toString(); + + // save to disc message.persist(err => { callback(err); }); @@ -783,6 +813,8 @@ function FTNMessageScanTossModule() { if(!_.isString(localNetworkName)) { next(new Error('No configuration for this packet')); } else { + + // :TODO: password needs validated - need to determine if it will use the same node config (which can have wildcards) or something else?! next(null); } @@ -1011,6 +1043,10 @@ FTNMessageScanTossModule.prototype.startup = function(cb) { if(exportSchedule.watchFile) { // :TODO: monitor file for changes/existance with gaze } + + if(_.isBoolean(exportSchedule.immediate)) { + this.exportImmediate = exportSchedule.immediate; + } } const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import); @@ -1043,6 +1079,10 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) { this.exportTimer.clear(); } + if(this.importTimer) { + this.importTimer.clear(); + } + // // Clean up temp dir/files we created // @@ -1087,7 +1127,8 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { // Select all messages that have a message_id > our last scan ID. // Additionally exclude messages that have a ftn_attr_flags FtnProperty meta // as those came via import! - // + // + /* const getNewUuidsSql = `SELECT message_id, message_uuid FROM message m @@ -1096,6 +1137,23 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { FROM message_meta WHERE message_id = m.message_id AND meta_category = 'FtnProperty' AND meta_name = 'ftn_attr_flags') = 0 ORDER BY message_id;`; + */ + + // + // Select all messages with a |message_id| > |lastScanId|. + // Additionally exclude messages with the System state_flags0 which will be present for + // imported or already exported messages + // + // NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here! + // + const getNewUuidsSql = + `SELECT message_id, message_uuid + FROM message m + WHERE area_tag = ? AND message_id > ? AND + (SELECT COUNT(message_id) + FROM message_meta + WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0 + ORDER BY message_id;`; var self = this; @@ -1156,5 +1214,26 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { FTNMessageScanTossModule.prototype.record = function(message) { // - // :TODO: If @immediate, we should do something here! + // This module works off schedules, but we do support @immediate for export + // + if(true !== this.exportImmediate || !this.hasValidConfiguration()) { + return; + } + + if(message.isPrivate()) { + // :TODO: support NetMail + } else if(message.areaTag) { + const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag]; + if(!this.isAreaConfigValid(areaConfig)) { + return; + } + + // :TODO: We must share a check to block export with schedule/timer when this is exporting... + // :TODO: Messages must be marked as "exported" else we will export this particular message again later @ schedule/timer + // ...if getNewUuidsSql in performExport checks for MSGID existence also we can omit already-exported messages + + this.exportMessagesToUplinks( [ message.uuid ], areaConfig, err => { + }); + + } };