* Resolve issue #59: Better message UUID generation and dupe checks

core/database.js

@@ -205,7 +205,6 @@ function createMessageBaseTables() {
 		END;`
 	);
 
-	//	:TODO: need SQL to ensure cleaned up if delete from message?
 	dbs.message.run(
 		`CREATE TABLE IF NOT EXISTS message_meta (
 			message_id		INTEGER NOT NULL,
@@ -213,11 +212,12 @@ function createMessageBaseTables() {
 			meta_name		VARCHAR NOT NULL,
 			meta_value		VARCHAR NOT NULL,
 			UNIQUE(message_id, meta_category, meta_name, meta_value), 
-			FOREIGN KEY(message_id) REFERENCES message(message_id)
+			FOREIGN KEY(message_id) REFERENCES message(message_id) ON DELETE CASCADE
 		);`
 	);
 
 	//	:TODO: need SQL to ensure cleaned up if delete from message?
+	/*
 	dbs.message.run(
 		`CREATE TABLE IF NOT EXISTS hash_tag (
 			hash_tag_id		INTEGER PRIMARY KEY,
@@ -230,9 +230,10 @@ function createMessageBaseTables() {
 	dbs.message.run(
 		`CREATE TABLE IF NOT EXISTS message_hash_tag (
 			hash_tag_id	INTEGER NOT NULL,
-			message_id	INTEGER NOT NULL
+			message_id	INTEGER NOT NULL,
 		);`
 	);
+	*/
 
 	dbs.message.run(
 		`CREATE TABLE IF NOT EXISTS user_message_area_last_read (

core/ftn_util.js

@@ -4,10 +4,8 @@
 let Config			= require('./config.js').config;
 let Address			= require('./ftn_address.js');
 let FNV1a			= require('./fnv1a.js');
-let createNamedUUID	= require('./uuid_util.js').createNamedUUID;
 
 let _				= require('lodash');
-let assert			= require('assert');
 let iconv			= require('iconv-lite');
 let moment			= require('moment');
 let uuid			= require('node-uuid');
@@ -18,8 +16,6 @@ let packageJson 	= require('../package.json');
 //	:TODO: Remove "Ftn" from most of these -- it's implied in the module
 exports.stringToNullPaddedBuffer	= stringToNullPaddedBuffer;
 exports.getMessageSerialNumber		= getMessageSerialNumber;
-exports.createMessageUuid			= createMessageUuid;
-exports.createMessageUuidAlternate	= createMessageUuidAlternate;
 exports.getDateFromFtnDateTime		= getDateFromFtnDateTime;
 exports.getDateTimeString			= getDateTimeString;
 
@@ -96,45 +92,6 @@ function getDateTimeString(m) {
 	return m.format('DD MMM YY  HH:mm:ss');
 }
 
-//
-//	Create a v5 named UUID given a message ID ("MSGID") and
-//	FTN area tag ("AREA").
-//
-//	This is similar to CrashMail
-//	See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
-//
-function createMessageUuid(ftnMsgId, ftnArea) {
-	assert(_.isString(ftnMsgId));
-	assert(_.isString(ftnArea));
-
-	ftnMsgId	= iconv.encode(ftnMsgId, 'CP437');
-	ftnArea		= iconv.encode(ftnArea.toUpperCase(), 'CP437');
-	
-	return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] )));
-};
-
-//
-//	Create a v5 named UUID given a FTN area tag ("AREA"),
-//	create/modified date, subject, and message body
-//
-//	This method should be used as a backup for when a MSGID is
-//	not available in which createMessageUuid() above should be
-//	used instead.
-//
-function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) {
-	assert(_.isString(ftnArea));
-	assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp));
-	assert(_.isString(subject));
-	assert(_.isString(msgBody));
-		
-	ftnArea			= iconv.encode(ftnArea.toUpperCase(), 'CP437');
-	modTimestamp	= iconv.encode(getDateTimeString(modTimestamp), 'CP437');
-	subject			= iconv.encode(subject.toUpperCase().trim(), 'CP437');
-	msgBody			= iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437');
-	
-	return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] )));
-}
-
 function getMessageSerialNumber(messageId) {
 	const msSinceEnigmaEpoc = (Date.now() - Date.UTC(2016, 1, 1));
 	const hash				= Math.abs(new FNV1a(msSinceEnigmaEpoc + messageId).value).toString(16);

core/message.js

@@ -4,21 +4,30 @@
 let msgDb			= require('./database.js').dbs.message;
 let wordWrapText	= require('./word_wrap.js').wordWrapText;
 let ftnUtil			= require('./ftn_util.js');
+let createNamedUUID	= require('./uuid_util.js').createNamedUUID;
 
 let uuid			= require('node-uuid');
 let async			= require('async');
 let _				= require('lodash');
 let assert			= require('assert');
 let moment			= require('moment');
+const iconvEncode	= require('iconv-lite').encode;
 
 module.exports = Message;
 
+const ENIGMA_MESSAGE_UUID_NAMESPACE 	= uuid.parse('154506df-1df8-46b9-98f8-ebb5815baaf8');
+
 function Message(options) {
 	options = options || {};
 
 	this.messageId		= options.messageId || 0;	//	always generated @ persist
 	this.areaTag		= options.areaTag || Message.WellKnownAreaTags.Invalid;
-	this.uuid			= options.uuid || uuid.v1();
+
+	if(options.uuid) {
+		//	note: new messages have UUID generated @ time of persist. See also Message.createMessageUUID()
+		this.uuid = options.uuid;
+	}
+
 	this.replyToMsgId	= options.replyToMsgId || 0;
 	this.toUserName		= options.toUserName || '';
 	this.fromUserName	= options.fromUserName || '';
@@ -110,6 +119,24 @@ Message.prototype.setLocalFromUserId = function(userId) {
 	this.meta.System.local_from_user_id = userId;
 };
 
+Message.createMessageUUID = function(areaTag, modTimestamp, subject, body) {
+	assert(_.isString(areaTag));
+	assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp));
+	assert(_.isString(subject));
+	assert(_.isString(body));
+
+	if(!moment.isMoment(modTimestamp)) {
+		modTimestamp = moment(modTimestamp);
+	}
+		
+	areaTag			= iconvEncode(areaTag.toUpperCase(), 'CP437');
+	modTimestamp	= iconvEncode(modTimestamp.format('DD MMM YY  HH:mm:ss'), 'CP437');
+	subject			= iconvEncode(subject.toUpperCase().trim(), 'CP437');
+	body			= iconvEncode(body.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437');
+	
+	return uuid.unparse(createNamedUUID(ENIGMA_MESSAGE_UUID_NAMESPACE, Buffer.concat( [ areaTag, modTimestamp, subject, body ] )));
+}
+
 Message.getMessageIdByUuid = function(uuid, cb) {
 	msgDb.get(
 		`SELECT message_id
@@ -330,10 +357,20 @@ Message.prototype.persist = function(cb) {
 				});
 			},
 			function storeMessage(callback) {
+				//	generate a UUID for this message if required (general case)
+				const msgTimestamp = moment();
+				if(!self.uuid) {
+					self.uuid = Message.createMessageUUID(
+						self.areaTag,
+						msgTimestamp,
+						self.subject,
+						self.message);
+				}
+
 				msgDb.run(
 					`INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp)
 					VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, 
-					[ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ],
+					[ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(msgTimestamp) ],
 					function inserted(err) {	//	use for this scope
 						if(!err) {
 							self.messageId = this.lastID;

core/scanner_tossers/ftn_bso.js

@@ -23,6 +23,7 @@ const assert				= require('assert');
 const gaze					= require('gaze');
 const fse					= require('fs-extra');
 const iconv					= require('iconv-lite');
+const uuid					= require('node-uuid');
 
 exports.moduleInfo = {
 	name	: 'FTN BSO',
@@ -783,9 +784,13 @@ function FTNMessageScanTossModule() {
 		}
 		
 		Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => {
-			if(msgIds && msgIds.length > 0) {
+			if(msgIds) {
-				assert(1 === msgIds.length);
+				//	expect a single match, but dupe checking is not perfect - warn otherwise
+				if(1 === msgIds.length) {
 					message.replyToMsgId = msgIds[0];	
+				} else {
+					Log.warn( { msgIds : msgIds, replyKludge :  message.meta.FtnKludge.REPLY }, 'Found 2:n MSGIDs matching REPLY kludge!');
+				}
 			}
 			cb();
 		});
@@ -804,24 +809,13 @@ function FTNMessageScanTossModule() {
 					message.areaTag = localAreaTag;
 					
 					//
-					//	If duplicates are NOT allowed in the area (the default), we need to update
+					//	If we *allow* dupes (disabled by default), then just generate
-					//	the message UUID using data available to us. Duplicate UUIDs are internally
+					//	a random UUID. Otherwise, don't assign the UUID just yet. It will be
-					//	not allowed in our local database.
+					//	generated at persist() time and should be consistent across import/exports
 					//
-					if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) {
+					if(Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) {
-						if(self.messageHasValidMSGID(message)) {
+						//	just generate a UUID & therefor always allow for dupes
-							//	Update UUID with our preferred generation method
+						message.uuid = uuid.v1();
-							message.uuid = ftnUtil.createMessageUuid(
-								message.meta.FtnKludge.MSGID,
-								message.meta.FtnProperty.ftn_area);
-						} else {
-							//	Update UUID with alternate/backup generation method
-							message.uuid = ftnUtil.createMessageUuidAlternate(
-								message.meta.FtnProperty.ftn_area, 
-								message.modTimestamp,
-								message.subject,
-								message.message);
-						}
 					}
 					
 					callback(null);	
@@ -847,6 +841,16 @@ function FTNMessageScanTossModule() {
 		);
 	};
 
+	this.appendTearAndOrigin = function(message) {
+		if(message.meta.FtnProperty.ftn_tear_line) {
+			message.message += `\r\n${message.meta.FtnProperty.ftn_tear_line}\r\n`;
+		}
+
+		if(message.meta.FtnProperty.ftn_origin) {
+			message.message += `${message.meta.FtnProperty.ftn_origin}\r\n`;
+		}
+	};
+	
 	//
 	//	Ref. implementations on import: 
 	//	*	https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c
@@ -855,7 +859,7 @@ function FTNMessageScanTossModule() {
 	this.importMessagesFromPacketFile = function(packetPath, password, cb) {
 		let packetHeader;
 				
-		const packetOpts = { keepTearAndOrigin : true };
+		const packetOpts = { keepTearAndOrigin : false };	//	needed so we can calc message UUID without these; we'll add later
 		
 		let importStats = {
 			areaSuccess	: {},	//	areaTag->count
@@ -886,6 +890,14 @@ function FTNMessageScanTossModule() {
 					//
 					const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag);
 					if(localAreaTag) {
+						message.uuid = Message.createMessageUUID(
+							localAreaTag,
+							message.modTimestamp,
+							message.subject,
+							message.message);
+
+						self.appendTearAndOrigin(message);
+						
 						self.importEchoMailToArea(localAreaTag, packetHeader, message, err => {
 							if(err) {
 								//	bump area fail stats