From afe0c88cfc14a515ed1c3aee7efd73910b0056ad Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 20 Jan 2018 15:16:35 -0700 Subject: [PATCH] NetMail non-HUB fixes * Properly separate FTN *packet* header vs *message* header DST/SRC information * Change routes{} handling: These are now *require* for out-of-HUB routing such that Enig will know where to send messages --- core/ftn_mail_packet.js | 97 ++++++++++++++++-------------- core/message.js | 8 +++ core/scanner_tossers/ftn_bso.js | 101 ++++++++++++++++---------------- 3 files changed, 113 insertions(+), 93 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index d2f003e9..1c60e1f0 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -524,13 +524,13 @@ function Packet(options) { ); }; - this.parsePacketMessages = function(packetBuffer, iterator, cb) { + this.parsePacketMessages = function(header, packetBuffer, iterator, cb) { binary.parse(packetBuffer) .word16lu('messageType') - .word16lu('ftn_orig_node') - .word16lu('ftn_dest_node') - .word16lu('ftn_orig_network') - .word16lu('ftn_dest_network') + .word16lu('ftn_msg_orig_node') + .word16lu('ftn_msg_dest_node') + .word16lu('ftn_msg_orig_net') + .word16lu('ftn_msg_dest_net') .word16lu('ftn_attr_flags') .word16lu('ftn_cost') .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max @@ -569,20 +569,28 @@ function Packet(options) { // contain an origin line, kludges, SAUCE in the case // of ANSI files, etc. // - let msg = new Message( { + const msg = new Message( { toUserName : convMsgData.toUserName, fromUserName : convMsgData.fromUserName, subject : convMsgData.subject, modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime), }); - msg.meta.FtnProperty = {}; - msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node; - msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node; - msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; - msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; - msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; - msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; + // :TODO: When non-private (e.g. EchoMail), attempt to extract SRC from MSGID vs headers, when avail (or Orgin line? research further) + msg.meta.FtnProperty = { + ftn_orig_node : header.origNode, + ftn_dest_node : header.destNode, + ftn_orig_network : header.origNet, + ftn_dest_network : header.destNet, + + ftn_attr_flags : msgData.ftn_attr_flags, + ftn_cost : msgData.ftn_cost, + + ftn_msg_orig_node : msgData.ftn_msg_orig_node, + ftn_msg_dest_node : msgData.ftn_msg_dest_node, + ftn_msg_orig_net : msgData.ftn_msg_orig_net, + ftn_msg_dest_net : msgData.ftn_msg_dest_net, + }; self.processMessageBody(msgData.message, messageBodyData => { msg.message = messageBodyData.message; @@ -622,11 +630,11 @@ function Packet(options) { const nextBuf = packetBuffer.slice(read); if(nextBuf.length > 0) { - let next = function(e) { + const next = function(e) { if(e) { cb(e); } else { - self.parsePacketMessages(nextBuf, iterator, cb); + self.parsePacketMessages(header, nextBuf, iterator, cb); } }; @@ -651,6 +659,10 @@ function Packet(options) { Message.FtnPropertyNames.FtnOrigPoint, Message.FtnPropertyNames.FtnDestPoint, Message.FtnPropertyNames.FtnAttribute, + Message.FtnPropertyNames.FtnMsgOrigNode, + Message.FtnPropertyNames.FtnMsgDestNode, + Message.FtnPropertyNames.FtnMsgOrigNet, + Message.FtnPropertyNames.FtnMsgDestNet, ].forEach( propName => { if(message.meta.FtnProperty[propName]) { message.meta.FtnProperty[propName] = parseInt(message.meta.FtnProperty[propName]) || 0; @@ -658,6 +670,25 @@ function Packet(options) { }); }; + this.writeMessageHeader = function(message, buf) { + // ensure address FtnProperties are numbers + self.sanatizeFtnProperties(message); + + const destNode = message.meta.FtnProperty.ftn_msg_dest_node || message.meta.FtnProperty.ftn_dest_node; + const destNet = message.meta.FtnProperty.ftn_msg_dest_net || message.meta.FtnProperty.ftn_dest_network; + + buf.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); + buf.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2); + buf.writeUInt16LE(destNode, 4); + buf.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); + buf.writeUInt16LE(destNet, 8); + buf.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); + buf.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); + + const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); + dateTimeBuffer.copy(buf, 14); + }; + this.getMessageEntryBuffer = function(message, options, cb) { function getAppendMeta(k, m, sepChar=':') { @@ -678,20 +709,7 @@ function Packet(options) { [ function prepareHeaderAndKludges(callback) { const basicHeader = new Buffer(34); - - // ensure address FtnProperties are numbers - self.sanatizeFtnProperties(message); - - basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); - - const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); - dateTimeBuffer.copy(basicHeader, 14); + self.writeMessageHeader(message, basicHeader); // // To, from, and subject must be NULL term'd and have max lengths as per spec. @@ -808,17 +826,7 @@ function Packet(options) { this.writeMessage = function(message, ws, options) { let basicHeader = new Buffer(34); - - basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); - - const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); - dateTimeBuffer.copy(basicHeader, 14); + self.writeMessageHeader(message, basicHeader); ws.write(basicHeader); @@ -911,7 +919,7 @@ function Packet(options) { }; this.parsePacketBuffer = function(packetBuffer, iterator, cb) { - async.series( + async.waterfall( [ function processHeader(callback) { self.parsePacketHeader(packetBuffer, (err, header) => { @@ -919,15 +927,16 @@ function Packet(options) { return callback(err); } - let next = function(e) { - callback(e); + const next = function(e) { + return callback(e, header); }; iterator('header', header, next); }); }, - function processMessages(callback) { + function processMessages(header, callback) { self.parsePacketMessages( + header, packetBuffer.slice(FTN_PACKET_HEADER_SIZE), iterator, callback); diff --git a/core/message.js b/core/message.js index 7cac5c79..f99efa8a 100644 --- a/core/message.js +++ b/core/message.js @@ -116,8 +116,10 @@ Message.StateFlags0 = { }; Message.FtnPropertyNames = { + // packet header oriented FtnOrigNode : 'ftn_orig_node', FtnDestNode : 'ftn_dest_node', + // :TODO: rename these to ftn_*_net vs network - ensure things won't break, may need mapping FtnOrigNetwork : 'ftn_orig_network', FtnDestNetwork : 'ftn_dest_network', FtnAttrFlags : 'ftn_attr_flags', @@ -127,6 +129,12 @@ Message.FtnPropertyNames = { FtnOrigPoint : 'ftn_orig_point', FtnDestPoint : 'ftn_dest_point', + // message header oriented + FtnMsgOrigNode : 'ftn_msg_orig_node', + FtnMsgDestNode : 'ftn_msg_dest_node', + FtnMsgOrigNet : 'ftn_msg_orig_net', + FtnMsgDestNet : 'ftn_msg_dest_net', + FtnAttribute : 'ftn_attribute', FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001 diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 20ab435f..36ee815f 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -306,12 +306,26 @@ function FTNMessageScanTossModule() { // const localAddress = new Address(options.network.localAddress); // ensure we have an Address obj not a string version + // :TODO: create Address.toMeta() / similar message.meta.FtnProperty = message.meta.FtnProperty || {}; message.meta.FtnKludge = message.meta.FtnKludge || {}; - message.meta.FtnProperty.ftn_orig_node = localAddress.node; - message.meta.FtnProperty.ftn_orig_network = localAddress.net; - message.meta.FtnProperty.ftn_cost = 0; + message.meta.FtnProperty.ftn_orig_node = localAddress.node; + message.meta.FtnProperty.ftn_orig_network = localAddress.net; + message.meta.FtnProperty.ftn_cost = 0; + message.meta.FtnProperty.ftn_msg_orig_node = localAddress.node; + message.meta.FtnProperty.ftn_msg_orig_net = localAddress.net; + + const destAddress = options.routeAddress || options.destAddress; + message.meta.FtnProperty.ftn_dest_node = destAddress.node; + message.meta.FtnProperty.ftn_dest_network = destAddress.net; + + if(destAddress.zone) { + message.meta.FtnProperty.ftn_dest_zone = destAddress.zone; + } + if(destAddress.point) { + message.meta.FtnProperty.ftn_dest_point = destAddress.point; + } // tear line and origin can both go in EchoMail & NetMail message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); @@ -320,9 +334,11 @@ function FTNMessageScanTossModule() { let ftnAttribute = ftnMailPacket.Packet.Attribute.Local; // message from our system if(self.isNetMailMessage(message)) { - // These should be set for Private/NetMail already - assert(_.isNumber(parseInt(message.meta.FtnProperty.ftn_dest_node))); - assert(_.isNumber(parseInt(message.meta.FtnProperty.ftn_dest_network))); + // + // Set route and message destination properties -- they may differ + // + message.meta.FtnProperty.ftn_msg_dest_node = options.destAddress.node; + message.meta.FtnProperty.ftn_msg_dest_net = options.destAddress.net; ftnAttribute |= ftnMailPacket.Packet.Attribute.Private; @@ -353,10 +369,6 @@ function FTNMessageScanTossModule() { message.meta.FtnKludge.TOPT = options.destAddress.point; } } else { - // We need to set some destination info for EchoMail - message.meta.FtnProperty.ftn_dest_node = options.destAddress.node; - message.meta.FtnProperty.ftn_dest_network = options.destAddress.net; - // // Set appropriate attribute flag for export type // @@ -573,7 +585,7 @@ function FTNMessageScanTossModule() { const packetHeader = new ftnMailPacket.PacketHeader( exportOpts.network.localAddress, - exportOpts.destAddress, + exportOpts.routeAddress, exportOpts.nodeConfig.packetType ); @@ -801,57 +813,44 @@ function FTNMessageScanTossModule() { return _.find(routes, (route, addrWildcard) => { return dstAddr.isPatternMatch(addrWildcard); }); - - /* - const route = _.find(routes, (route, addrWildcard) => { - return dstAddr.isPatternMatch(addrWildcard); - }); - - if(route && route.address) { - return Address.fromString(route.address); - } - */ }; - this.getAcceptableNetMailNetworkInfoFromAddress = function(dstAddr, cb) { + this.getNetMailRouteInfoFromAddress = function(destAddress, cb) { // - // Attempt to find an acceptable network configuration using the following - // lookup order (most to least explicit config): + // Attempt to find route information for |destAddress|: // // 1) Routes: messageNetworks.ftn.netMail.routes{} -> scannerTossers.ftn_bso.nodes{} -> config - // - Where we send may not be where dstAddress is (it's routed!) + // - Where we send may not be where destAddress is (it's routed!) // 2) Direct to nodes: scannerTossers.ftn_bso.nodes{} -> config - // - Where we send is direct to dstAddr + // - Where we send is direct to destAddress // // In both cases, attempt to look up Zone:Net/* to discover local "from" network/address // falling back to Config.scannerTossers.ftn_bso.defaultNetwork // - const route = this.getNetMailRoute(dstAddr); + const route = this.getNetMailRoute(destAddress); let routeAddress; let networkName; + let isRouted; if(route) { routeAddress = Address.fromString(route.address); networkName = route.network; + isRouted = true; } else { - routeAddress = dstAddr; + routeAddress = destAddress; + isRouted = false; } - networkName = networkName || - this.getNetworkNameByAddressPattern(`${routeAddress.zone}:${routeAddress.net}/*`) || - Config.scannerTossers.ftn_bso.defaultNetwork - ; + networkName = networkName || this.getNetworkNameByAddress(routeAddress); const config = _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => { return routeAddress.isPatternMatch(nodeAddrWildcard); - }) || { - packetType : '2+', - encoding : Config.scannerTossers.ftn_bso.packetMsgEncoding, - }; + }) || { packetType : '2+', encoding : Config.scannerTossers.ftn_bso.packetMsgEncoding }; + // we should never be failing here; we may just be using defaults. return cb( - config ? null : Errors.DoesNotExist(`No configuration found for ${dstAddr.toString()}`), - config, routeAddress, networkName + networkName ? null : Errors.DoesNotExist(`No NetMail route for ${destAddress.toString()}`), + { destAddress, routeAddress, networkName, config, isRouted } ); }; @@ -876,21 +875,22 @@ function FTNMessageScanTossModule() { function discoverUplink(callback) { const dstAddr = new Address(message.meta.System[Message.SystemMetaNames.RemoteToUser]); - return self.getAcceptableNetMailNetworkInfoFromAddress(dstAddr, (err, config, routeAddress, networkName) => { + self.getNetMailRouteInfoFromAddress(dstAddr, (err, routeInfo) => { if(err) { return callback(err); } - exportOpts.nodeConfig = config; - exportOpts.destAddress = routeAddress; - exportOpts.fileCase = config.fileCase || 'lower'; - exportOpts.network = Config.messageNetworks.ftn.networks[networkName]; - exportOpts.networkName = networkName; + exportOpts.nodeConfig = routeInfo.config; + exportOpts.destAddress = dstAddr; + exportOpts.routeAddress = routeInfo.routeAddress; + exportOpts.fileCase = routeInfo.config.fileCase || 'lower'; + exportOpts.network = Config.messageNetworks.ftn.networks[routeInfo.networkName]; + exportOpts.networkName = routeInfo.networkName; exportOpts.outgoingDir = self.getOutgoingEchoMailPacketDir(exportOpts.networkName, exportOpts.destAddress); - exportOpts.exportType = self.getExportType(config); + exportOpts.exportType = self.getExportType(routeInfo.config); if(!exportOpts.network) { - return callback(Errors.DoesNotExist(`No configuration found for network ${networkName}`)); + return callback(Errors.DoesNotExist(`No configuration found for network ${routeInfo.networkName}`)); } return callback(null); @@ -937,12 +937,15 @@ function FTNMessageScanTossModule() { ], err => { if(err) { - Log.warn( { error :err.message }, 'Error exporting message' ); + Log.warn( { error : err.message }, 'Error exporting message' ); } return nextMessageOrUuid(null); } ); }, err => { + if(err) { + Log.warn( { error : err.message }, 'Error(s) during NetMail export'); + } return cb(err); }); }; @@ -962,6 +965,7 @@ function FTNMessageScanTossModule() { fileCase : self.moduleConfig.nodes[nodeConfigKey].fileCase || 'lower', }; + if(_.isString(exportOpts.network.localAddress)) { exportOpts.network.localAddress = Address.fromString(exportOpts.network.localAddress); } @@ -2031,8 +2035,7 @@ function FTNMessageScanTossModule() { this.isNetMailMessage = function(message) { return message.isPrivate() && null === _.get(message, 'meta.System.LocalToUserID', null) && - Message.AddressFlavor.FTN === _.get(message, 'meta.System.external_flavor', null) - ; + Message.AddressFlavor.FTN === _.get(message, 'meta.System.external_flavor', null); }; }