diff --git a/core/database.js b/core/database.js index 9c5daac9..924b60ed 100644 --- a/core/database.js +++ b/core/database.js @@ -205,7 +205,6 @@ function createMessageBaseTables() { END;` ); - // :TODO: need SQL to ensure cleaned up if delete from message? dbs.message.run( `CREATE TABLE IF NOT EXISTS message_meta ( message_id INTEGER NOT NULL, @@ -213,11 +212,12 @@ function createMessageBaseTables() { meta_name VARCHAR NOT NULL, meta_value VARCHAR NOT NULL, UNIQUE(message_id, meta_category, meta_name, meta_value), - FOREIGN KEY(message_id) REFERENCES message(message_id) + FOREIGN KEY(message_id) REFERENCES message(message_id) ON DELETE CASCADE );` ); // :TODO: need SQL to ensure cleaned up if delete from message? + /* dbs.message.run( `CREATE TABLE IF NOT EXISTS hash_tag ( hash_tag_id INTEGER PRIMARY KEY, @@ -230,9 +230,10 @@ function createMessageBaseTables() { dbs.message.run( `CREATE TABLE IF NOT EXISTS message_hash_tag ( hash_tag_id INTEGER NOT NULL, - message_id INTEGER NOT NULL + message_id INTEGER NOT NULL, );` ); + */ dbs.message.run( `CREATE TABLE IF NOT EXISTS user_message_area_last_read ( diff --git a/core/ftn_util.js b/core/ftn_util.js index 11fb9a15..0860aeb4 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -4,10 +4,8 @@ let Config = require('./config.js').config; let Address = require('./ftn_address.js'); let FNV1a = require('./fnv1a.js'); -let createNamedUUID = require('./uuid_util.js').createNamedUUID; let _ = require('lodash'); -let assert = require('assert'); let iconv = require('iconv-lite'); let moment = require('moment'); let uuid = require('node-uuid'); @@ -18,8 +16,6 @@ let packageJson = require('../package.json'); // :TODO: Remove "Ftn" from most of these -- it's implied in the module exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; exports.getMessageSerialNumber = getMessageSerialNumber; -exports.createMessageUuid = createMessageUuid; -exports.createMessageUuidAlternate = createMessageUuidAlternate; exports.getDateFromFtnDateTime = getDateFromFtnDateTime; exports.getDateTimeString = getDateTimeString; @@ -96,45 +92,6 @@ function getDateTimeString(m) { return m.format('DD MMM YY HH:mm:ss'); } -// -// Create a v5 named UUID given a message ID ("MSGID") and -// FTN area tag ("AREA"). -// -// This is similar to CrashMail -// See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c -// -function createMessageUuid(ftnMsgId, ftnArea) { - assert(_.isString(ftnMsgId)); - assert(_.isString(ftnArea)); - - ftnMsgId = iconv.encode(ftnMsgId, 'CP437'); - ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437'); - - return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] ))); -}; - -// -// Create a v5 named UUID given a FTN area tag ("AREA"), -// create/modified date, subject, and message body -// -// This method should be used as a backup for when a MSGID is -// not available in which createMessageUuid() above should be -// used instead. -// -function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) { - assert(_.isString(ftnArea)); - assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp)); - assert(_.isString(subject)); - assert(_.isString(msgBody)); - - ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437'); - modTimestamp = iconv.encode(getDateTimeString(modTimestamp), 'CP437'); - subject = iconv.encode(subject.toUpperCase().trim(), 'CP437'); - msgBody = iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437'); - - return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] ))); -} - function getMessageSerialNumber(messageId) { const msSinceEnigmaEpoc = (Date.now() - Date.UTC(2016, 1, 1)); const hash = Math.abs(new FNV1a(msSinceEnigmaEpoc + messageId).value).toString(16); diff --git a/core/message.js b/core/message.js index 9bffe415..82536349 100644 --- a/core/message.js +++ b/core/message.js @@ -4,21 +4,30 @@ let msgDb = require('./database.js').dbs.message; let wordWrapText = require('./word_wrap.js').wordWrapText; let ftnUtil = require('./ftn_util.js'); +let createNamedUUID = require('./uuid_util.js').createNamedUUID; let uuid = require('node-uuid'); let async = require('async'); let _ = require('lodash'); let assert = require('assert'); let moment = require('moment'); +const iconvEncode = require('iconv-lite').encode; module.exports = Message; +const ENIGMA_MESSAGE_UUID_NAMESPACE = uuid.parse('154506df-1df8-46b9-98f8-ebb5815baaf8'); + function Message(options) { options = options || {}; this.messageId = options.messageId || 0; // always generated @ persist this.areaTag = options.areaTag || Message.WellKnownAreaTags.Invalid; - this.uuid = options.uuid || uuid.v1(); + + if(options.uuid) { + // note: new messages have UUID generated @ time of persist. See also Message.createMessageUUID() + this.uuid = options.uuid; + } + this.replyToMsgId = options.replyToMsgId || 0; this.toUserName = options.toUserName || ''; this.fromUserName = options.fromUserName || ''; @@ -110,6 +119,24 @@ Message.prototype.setLocalFromUserId = function(userId) { this.meta.System.local_from_user_id = userId; }; +Message.createMessageUUID = function(areaTag, modTimestamp, subject, body) { + assert(_.isString(areaTag)); + assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp)); + assert(_.isString(subject)); + assert(_.isString(body)); + + if(!moment.isMoment(modTimestamp)) { + modTimestamp = moment(modTimestamp); + } + + areaTag = iconvEncode(areaTag.toUpperCase(), 'CP437'); + modTimestamp = iconvEncode(modTimestamp.format('DD MMM YY HH:mm:ss'), 'CP437'); + subject = iconvEncode(subject.toUpperCase().trim(), 'CP437'); + body = iconvEncode(body.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437'); + + return uuid.unparse(createNamedUUID(ENIGMA_MESSAGE_UUID_NAMESPACE, Buffer.concat( [ areaTag, modTimestamp, subject, body ] ))); +} + Message.getMessageIdByUuid = function(uuid, cb) { msgDb.get( `SELECT message_id @@ -330,10 +357,20 @@ Message.prototype.persist = function(cb) { }); }, function storeMessage(callback) { + // generate a UUID for this message if required (general case) + const msgTimestamp = moment(); + if(!self.uuid) { + self.uuid = Message.createMessageUUID( + self.areaTag, + msgTimestamp, + self.subject, + self.message); + } + msgDb.run( `INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, - [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], + [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(msgTimestamp) ], function inserted(err) { // use for this scope if(!err) { self.messageId = this.lastID; diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 030ce380..b5e34985 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -23,6 +23,7 @@ const assert = require('assert'); const gaze = require('gaze'); const fse = require('fs-extra'); const iconv = require('iconv-lite'); +const uuid = require('node-uuid'); exports.moduleInfo = { name : 'FTN BSO', @@ -192,11 +193,11 @@ function FTNMessageScanTossModule() { let ext; switch(flowType) { - case 'mail' : ext = `${exportType.toLowerCase()[0]}ut`; break; - case 'ref' : ext = `${exportType.toLowerCase()[0]}lo`; break; - case 'busy' : ext = 'bsy'; break; - case 'request' : ext = 'req'; break; - case 'requests' : ext = 'hrq'; break; + case 'mail' : ext = `${exportType.toLowerCase()[0]}ut`; break; + case 'ref' : ext = `${exportType.toLowerCase()[0]}lo`; break; + case 'busy' : ext = 'bsy'; break; + case 'request' : ext = 'req'; break; + case 'requests' : ext = 'hrq'; break; } return ext; @@ -307,8 +308,8 @@ function FTNMessageScanTossModule() { // Set appropriate attribute flag for export type // switch(this.getExportType(options.nodeConfig)) { - case 'crash' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Crash; break; - case 'hold' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Hold; break; + case 'crash' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Crash; break; + case 'hold' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Hold; break; // :TODO: Others? } @@ -783,9 +784,13 @@ function FTNMessageScanTossModule() { } Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => { - if(msgIds && msgIds.length > 0) { - assert(1 === msgIds.length); - message.replyToMsgId = msgIds[0]; + if(msgIds) { + // expect a single match, but dupe checking is not perfect - warn otherwise + if(1 === msgIds.length) { + message.replyToMsgId = msgIds[0]; + } else { + Log.warn( { msgIds : msgIds, replyKludge : message.meta.FtnKludge.REPLY }, 'Found 2:n MSGIDs matching REPLY kludge!'); + } } cb(); }); @@ -804,24 +809,13 @@ function FTNMessageScanTossModule() { message.areaTag = localAreaTag; // - // If duplicates are NOT allowed in the area (the default), we need to update - // the message UUID using data available to us. Duplicate UUIDs are internally - // not allowed in our local database. - // - if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) { - if(self.messageHasValidMSGID(message)) { - // Update UUID with our preferred generation method - message.uuid = ftnUtil.createMessageUuid( - message.meta.FtnKludge.MSGID, - message.meta.FtnProperty.ftn_area); - } else { - // Update UUID with alternate/backup generation method - message.uuid = ftnUtil.createMessageUuidAlternate( - message.meta.FtnProperty.ftn_area, - message.modTimestamp, - message.subject, - message.message); - } + // If we *allow* dupes (disabled by default), then just generate + // a random UUID. Otherwise, don't assign the UUID just yet. It will be + // generated at persist() time and should be consistent across import/exports + // + if(Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) { + // just generate a UUID & therefor always allow for dupes + message.uuid = uuid.v1(); } callback(null); @@ -846,6 +840,16 @@ function FTNMessageScanTossModule() { } ); }; + + this.appendTearAndOrigin = function(message) { + if(message.meta.FtnProperty.ftn_tear_line) { + message.message += `\r\n${message.meta.FtnProperty.ftn_tear_line}\r\n`; + } + + if(message.meta.FtnProperty.ftn_origin) { + message.message += `${message.meta.FtnProperty.ftn_origin}\r\n`; + } + }; // // Ref. implementations on import: @@ -855,7 +859,7 @@ function FTNMessageScanTossModule() { this.importMessagesFromPacketFile = function(packetPath, password, cb) { let packetHeader; - const packetOpts = { keepTearAndOrigin : true }; + const packetOpts = { keepTearAndOrigin : false }; // needed so we can calc message UUID without these; we'll add later let importStats = { areaSuccess : {}, // areaTag->count @@ -879,13 +883,21 @@ function FTNMessageScanTossModule() { } else if('message' === entryType) { const message = entryData; const areaTag = message.meta.FtnProperty.ftn_area; - + if(areaTag) { // // EchoMail // const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag); if(localAreaTag) { + message.uuid = Message.createMessageUUID( + localAreaTag, + message.modTimestamp, + message.subject, + message.message); + + self.appendTearAndOrigin(message); + self.importEchoMailToArea(localAreaTag, packetHeader, message, err => { if(err) { // bump area fail stats