From 317af8419a045db979907fdc008be31fc47f2ab3 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 2 Feb 2016 21:35:59 -0700 Subject: [PATCH 01/27] Major commit for new message network WIP --- README.md | 2 +- core/acs_parser.js | 31 +- core/acs_util.js | 5 + core/art.js | 157 +--- core/asset.js | 1 - core/bbs.js | 10 +- core/config.js | 62 +- core/database.js | 6 +- core/fse.js | 32 +- core/ftn_mail_packet.js | 742 +++++++++++++++++-- core/ftn_util.js | 36 + core/menu_module.js | 9 +- core/menu_stack.js | 2 +- core/menu_util.js | 97 +-- core/message.js | 23 +- core/message_area.js | 328 ++++++-- core/module_util.js | 41 +- core/msg_network_module.js | 25 + core/msg_networks/ftn_msg_network_module.js | 27 + core/new_scan.js | 109 ++- core/predefined_mci.js | 13 +- core/sauce.js | 165 +++++ core/standard_menu.js | 4 +- core/theme.js | 16 +- core/user_login.js | 2 +- core/vertical_menu_view.js | 13 +- core/view_controller.js | 1 - misc/acs_parser.pegjs | 32 +- mods/abracadabra.js | 6 - mods/menu.hjson | 41 +- mods/msg_area_list.js | 75 +- mods/msg_area_post_fse.js | 8 +- mods/msg_area_view_fse.js | 2 +- mods/msg_conf_list.js | 122 +++ mods/msg_list.js | 41 +- mods/nua.js | 26 +- mods/themes/luciano_blocktronics/theme.hjson | 17 +- mods/user_list.js | 2 +- mods/whos_online.js | 1 - oputil.js | 14 +- 40 files changed, 1747 insertions(+), 599 deletions(-) create mode 100644 core/msg_network_module.js create mode 100644 core/msg_networks/ftn_msg_network_module.js create mode 100644 core/sauce.js create mode 100644 mods/msg_conf_list.js diff --git a/README.md b/README.md index 9ed90948..072633a8 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Please see the [Quickstart](docs/index.md#quickstart) ## License Released under the [BSD 2-clause](https://opensource.org/licenses/BSD-2-Clause) license: -Copyright (c) 2015, Bryan D. Ashby +Copyright (c) 2015-2016, Bryan D. Ashby All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/core/acs_parser.js b/core/acs_parser.js index 874033db..454e1ba5 100644 --- a/core/acs_parser.js +++ b/core/acs_parser.js @@ -804,16 +804,13 @@ module.exports = (function() { return !isNaN(value) && user.getAge() >= value; }, AS : function accountStatus() { - - if(_.isNumber(value)) { + if(!_.isArray(value)) { value = [ value ]; } - assert(_.isArray(value)); - - return _.findIndex(value, function cmp(accStatus) { - return parseInt(accStatus, 10) === parseInt(user.properties.account_status, 10); - }) > -1; + const userAccountStatus = parseInt(user.properties.account_status, 10); + value = value.map(n => parseInt(n, 10)); // ensure we have integers + return value.indexOf(userAccountStatus) > -1; }, EC : function isEncoding() { switch(value) { @@ -842,7 +839,7 @@ module.exports = (function() { // :TODO: implement me!! return false; }, - SC : function isSecerConnection() { + SC : function isSecureConnection() { return client.session.isSecure; }, ML : function minutesLeft() { @@ -870,16 +867,20 @@ module.exports = (function() { return !isNaN(value) && client.term.termWidth >= value; }, ID : function isUserId(value) { - return user.userId === value; + if(!_.isArray(value)) { + value = [ value ]; + } + + value = value.map(n => parseInt(n, 10)); // ensure we have integers + return value.indexOf(user.userId) > -1; }, WD : function isOneOfDayOfWeek() { - // :TODO: return true if DoW - if(_.isNumber(value)) { - - } else if(_.isArray(value)) { - + if(!_.isArray(value)) { + value = [ value ]; } - return false; + + value = value.map(n => parseInt(n, 10)); // ensure we have integers + return value.indexOf(new Date().getDay()) > -1; }, MM : function isMinutesPastMidnight() { // :TODO: return true if value is >= minutes past midnight sys time diff --git a/core/acs_util.js b/core/acs_util.js index fe111fe1..0f91927d 100644 --- a/core/acs_util.js +++ b/core/acs_util.js @@ -7,8 +7,13 @@ var acsParser = require('./acs_parser.js'); var _ = require('lodash'); var assert = require('assert'); +exports.checkAcs = checkAcs; exports.getConditionalValue = getConditionalValue; +function checkAcs(client, acsString) { + return acsParser.parse(acsString, { client : client } ); +} + function getConditionalValue(client, condArray, memberName) { assert(_.isObject(client)); assert(_.isArray(condArray)); diff --git a/core/art.js b/core/art.js index 9cf0dd9b..5efdbdee 100644 --- a/core/art.js +++ b/core/art.js @@ -12,6 +12,7 @@ var events = require('events'); var util = require('util'); var ansi = require('./ansi_term.js'); var aep = require('./ansi_escape_parser.js'); +var sauce = require('./sauce.js'); var _ = require('lodash'); @@ -20,10 +21,6 @@ exports.getArtFromPath = getArtFromPath; exports.display = display; exports.defaultEncodingFromExtension = defaultEncodingFromExtension; -var SAUCE_SIZE = 128; -var SAUCE_ID = new Buffer([0x53, 0x41, 0x55, 0x43, 0x45]); // 'SAUCE' -var COMNT_ID = new Buffer([0x43, 0x4f, 0x4d, 0x4e, 0x54]); // 'COMNT' - // :TODO: Return MCI code information // :TODO: process SAUCE comments // :TODO: return font + font mapped information from SAUCE @@ -43,156 +40,6 @@ var SUPPORTED_ART_TYPES = { // :TODO: extension for topaz ansi/ascii. }; -// -// See -// http://www.acid.org/info/sauce/sauce.htm -// -// :TODO: Move all SAUCE stuff to sauce.js -function readSAUCE(data, cb) { - if(data.length < SAUCE_SIZE) { - cb(new Error('No SAUCE record present')); - return; - } - - var offset = data.length - SAUCE_SIZE; - var sauceRec = data.slice(offset); - - binary.parse(sauceRec) - .buffer('id', 5) - .buffer('version', 2) - .buffer('title', 35) - .buffer('author', 20) - .buffer('group', 20) - .buffer('date', 8) - .word32lu('fileSize') - .word8('dataType') - .word8('fileType') - .word16lu('tinfo1') - .word16lu('tinfo2') - .word16lu('tinfo3') - .word16lu('tinfo4') - .word8('numComments') - .word8('flags') - .buffer('tinfos', 22) // SAUCE 00.5 - .tap(function onVars(vars) { - - if(!SAUCE_ID.equals(vars.id)) { - cb(new Error('No SAUCE record present')); - return; - } - - var ver = iconv.decode(vars.version, 'cp437'); - - if('00' !== ver) { - cb(new Error('Unsupported SAUCE version: ' + ver)); - return; - } - - var sauce = { - id : iconv.decode(vars.id, 'cp437'), - version : iconv.decode(vars.version, 'cp437').trim(), - title : iconv.decode(vars.title, 'cp437').trim(), - author : iconv.decode(vars.author, 'cp437').trim(), - group : iconv.decode(vars.group, 'cp437').trim(), - date : iconv.decode(vars.date, 'cp437').trim(), - fileSize : vars.fileSize, - dataType : vars.dataType, - fileType : vars.fileType, - tinfo1 : vars.tinfo1, - tinfo2 : vars.tinfo2, - tinfo3 : vars.tinfo3, - tinfo4 : vars.tinfo4, - numComments : vars.numComments, - flags : vars.flags, - tinfos : vars.tinfos, - }; - - var dt = SAUCE_DATA_TYPES[sauce.dataType]; - if(dt && dt.parser) { - sauce[dt.name] = dt.parser(sauce); - } - - cb(null, sauce); - }); -} - -// :TODO: These need completed: -var SAUCE_DATA_TYPES = {}; -SAUCE_DATA_TYPES[0] = { name : 'None' }; -SAUCE_DATA_TYPES[1] = { name : 'Character', parser : parseCharacterSAUCE }; -SAUCE_DATA_TYPES[2] = 'Bitmap'; -SAUCE_DATA_TYPES[3] = 'Vector'; -SAUCE_DATA_TYPES[4] = 'Audio'; -SAUCE_DATA_TYPES[5] = 'BinaryText'; -SAUCE_DATA_TYPES[6] = 'XBin'; -SAUCE_DATA_TYPES[7] = 'Archive'; -SAUCE_DATA_TYPES[8] = 'Executable'; - -var SAUCE_CHARACTER_FILE_TYPES = {}; -SAUCE_CHARACTER_FILE_TYPES[0] = 'ASCII'; -SAUCE_CHARACTER_FILE_TYPES[1] = 'ANSi'; -SAUCE_CHARACTER_FILE_TYPES[2] = 'ANSiMation'; -SAUCE_CHARACTER_FILE_TYPES[3] = 'RIP script'; -SAUCE_CHARACTER_FILE_TYPES[4] = 'PCBoard'; -SAUCE_CHARACTER_FILE_TYPES[5] = 'Avatar'; -SAUCE_CHARACTER_FILE_TYPES[6] = 'HTML'; -SAUCE_CHARACTER_FILE_TYPES[7] = 'Source'; -SAUCE_CHARACTER_FILE_TYPES[8] = 'TundraDraw'; - -// -// Map of SAUCE font -> encoding hint -// -// Note that this is the same mapping that x84 uses. Be compatible! -// -var SAUCE_FONT_TO_ENCODING_HINT = { - 'Amiga MicroKnight' : 'amiga', - 'Amiga MicroKnight+' : 'amiga', - 'Amiga mOsOul' : 'amiga', - 'Amiga P0T-NOoDLE' : 'amiga', - 'Amiga Topaz 1' : 'amiga', - 'Amiga Topaz 1+' : 'amiga', - 'Amiga Topaz 2' : 'amiga', - 'Amiga Topaz 2+' : 'amiga', - 'Atari ATASCII' : 'atari', - 'IBM EGA43' : 'cp437', - 'IBM EGA' : 'cp437', - 'IBM VGA25G' : 'cp437', - 'IBM VGA50' : 'cp437', - 'IBM VGA' : 'cp437', -}; - -['437', '720', '737', '775', '819', '850', '852', '855', '857', '858', -'860', '861', '862', '863', '864', '865', '866', '869', '872'].forEach(function onPage(page) { - var codec = 'cp' + page; - SAUCE_FONT_TO_ENCODING_HINT['IBM EGA43 ' + page] = codec; - SAUCE_FONT_TO_ENCODING_HINT['IBM EGA ' + page] = codec; - SAUCE_FONT_TO_ENCODING_HINT['IBM VGA25g ' + page] = codec; - SAUCE_FONT_TO_ENCODING_HINT['IBM VGA50 ' + page] = codec; - SAUCE_FONT_TO_ENCODING_HINT['IBM VGA ' + page] = codec; -}); - -function parseCharacterSAUCE(sauce) { - var result = {}; - - result.fileType = SAUCE_CHARACTER_FILE_TYPES[sauce.fileType] || 'Unknown'; - - if(sauce.fileType === 0 || sauce.fileType === 1 || sauce.fileType === 2) { - // convience: create ansiFlags - sauce.ansiFlags = sauce.flags; - - var i = 0; - while(i < sauce.tinfos.length && sauce.tinfos[i] !== 0x00) { - ++i; - } - var fontName = iconv.decode(sauce.tinfos.slice(0, i), 'cp437'); - if(fontName.length > 0) { - result.fontName = fontName; - } - } - - return result; -} - function getFontNameFromSAUCE(sauce) { if(sauce.Character) { return sauce.Character.fontName; @@ -249,7 +96,7 @@ function getArtFromPath(path, options, cb) { } if(options.readSauce === true) { - readSAUCE(data, function onSauce(err, sauce) { + sauce.readSAUCE(data, function onSauce(err, sauce) { if(err) { cb(null, getResult()); } else { diff --git a/core/asset.js b/core/asset.js index 7f25a683..566add7c 100644 --- a/core/asset.js +++ b/core/asset.js @@ -2,7 +2,6 @@ 'use strict'; var Config = require('./config.js').config; -var theme = require('./theme.js'); var _ = require('lodash'); var assert = require('assert'); diff --git a/core/bbs.js b/core/bbs.js index 92b0e094..5c61e014 100644 --- a/core/bbs.js +++ b/core/bbs.js @@ -1,6 +1,9 @@ /* jslint node: true */ 'use strict'; +//var SegfaultHandler = require('segfault-handler'); +//SegfaultHandler.registerHandler('enigma-bbs-segfault.log'); + // ENiGMA½ var conf = require('./config.js'); var logger = require('./logger.js'); @@ -21,7 +24,7 @@ function bbsMain() { async.waterfall( [ function processArgs(callback) { - var args = parseArgs(); + const args = parseArgs(); var configPath; @@ -37,8 +40,7 @@ function bbsMain() { } } - var configPathSupplied = _.isString(configPath); - callback(null, configPath || conf.getDefaultPath(), configPathSupplied); + callback(null, configPath || conf.getDefaultPath(), _.isString(configPath)); }, function initConfig(configPath, configPathSupplied, callback) { conf.init(configPath, function configInit(err) { @@ -117,7 +119,7 @@ function initialize(cb) { process.exit(); }); - + // Init some extensions require('string-format').extend(String.prototype, require('./string_util.js').stringFormatExtensions); diff --git a/core/config.js b/core/config.js index 6bc9ac61..dc8f3a8e 100644 --- a/core/config.js +++ b/core/config.js @@ -8,10 +8,37 @@ var paths = require('path'); var async = require('async'); var _ = require('lodash'); var hjson = require('hjson'); +var assert = require('assert'); exports.init = init; exports.getDefaultPath = getDefaultPath; +function hasMessageConferenceAndArea(config) { + assert(_.isObject(config.messageConferences)); // we create one ourself! + + const nonInternalConfs = Object.keys(config.messageConferences).filter(confTag => { + return 'system_internal' !== confTag; + }); + + if(0 === nonInternalConfs.length) { + return false; + } + + // :TODO: there is likely a better/cleaner way of doing this + + var result = false; + _.forEach(nonInternalConfs, confTag => { + if(_.has(config.messageConferences[confTag], 'areas') && + Object.keys(config.messageConferences[confTag].areas) > 0) + { + result = true; + return false; // stop iteration + } + }); + + return result; +} + function init(configPath, cb) { async.waterfall( [ @@ -48,18 +75,13 @@ function init(configPath, cb) { // // Various sections must now exist in config // - if(!_.has(mergedConfig, 'messages.areas.') || - !_.isArray(mergedConfig.messages.areas) || - 0 === mergedConfig.messages.areas.length || - !_.isString(mergedConfig.messages.areas[0].name)) - { - var msgAreasErr = new Error('Please create at least one message area'); + if(hasMessageConferenceAndArea(mergedConfig)) { + var msgAreasErr = new Error('Please create at least one message conference and area!'); msgAreasErr.code = 'EBADCONFIG'; callback(msgAreasErr); - return; - } - - callback(null, mergedConfig); + } else { + callback(null, mergedConfig); + } } ], function complete(err, mergedConfig) { @@ -150,6 +172,7 @@ function getDefaultConfig() { paths : { mods : paths.join(__dirname, './../mods/'), servers : paths.join(__dirname, './servers/'), + msgNetworks : paths.join(__dirname, './msg_networks/'), art : paths.join(__dirname, './../mods/art/'), themes : paths.join(__dirname, './../mods/themes/'), logs : paths.join(__dirname, './../logs/'), // :TODO: set up based on system, e.g. /var/logs/enigmabbs or such @@ -183,6 +206,25 @@ function getDefaultConfig() { } }, + messageConferences : { + system_internal : { + name : 'System Internal', + desc : 'Built in conference for private messages, bulletins, etc.', + + areas : { + private_mail : { + name : 'Private Mail', + desc : 'Private user to user mail/email', + }, + + local_bulletin : { + name : 'System Bulletins', + desc : 'Bulletin messages for all users', + } + } + } + }, + messages : { areas : [ { name : 'private_mail', desc : 'Private Email', groups : [ 'users' ] } diff --git a/core/database.js b/core/database.js index 1b8f9afa..cb1a97f4 100644 --- a/core/database.js +++ b/core/database.js @@ -132,7 +132,7 @@ function createMessageBaseTables() { dbs.message.run( 'CREATE TABLE IF NOT EXISTS message (' + ' message_id INTEGER PRIMARY KEY,' + - ' area_name VARCHAR NOT NULL,' + + ' area_tag VARCHAR NOT NULL,' + ' message_uuid VARCHAR(36) NOT NULL,' + ' reply_to_message_id INTEGER,' + ' to_user_name VARCHAR NOT NULL,' + @@ -198,9 +198,9 @@ function createMessageBaseTables() { dbs.message.run( 'CREATE TABLE IF NOT EXISTS user_message_area_last_read (' + ' user_id INTEGER NOT NULL,' + - ' area_name VARCHAR NOT NULL,' + + ' area_tag VARCHAR NOT NULL,' + ' message_id INTEGER NOT NULL,' + - ' UNIQUE(user_id, area_name)' + + ' UNIQUE(user_id, area_tag)' + ');' ); diff --git a/core/fse.js b/core/fse.js index a8cc6843..69c5e418 100644 --- a/core/fse.js +++ b/core/fse.js @@ -7,7 +7,7 @@ var ansi = require('../core/ansi_term.js'); var theme = require('../core/theme.js'); var MultiLineEditTextView = require('../core/multi_line_edit_text_view.js').MultiLineEditTextView; var Message = require('../core/message.js'); -var getMessageAreaByName = require('../core/message_area.js').getMessageAreaByName; +var getMessageAreaByTag = require('../core/message_area.js').getMessageAreaByTag; var updateMessageAreaLastReadId = require('../core/message_area.js').updateMessageAreaLastReadId; var getUserIdAndName = require('../core/user.js').getUserIdAndName; @@ -75,6 +75,8 @@ var MCICodeIds = { HashTags : 9, MessageID : 10, ReplyToMsgID : 11, + + // :TODO: ConfName }, @@ -104,15 +106,15 @@ function FullScreenEditorModule(options) { // editorMode : view | edit | quote // // menuConfig.config or extraArgs - // messageAreaName + // messageAreaTag // messageIndex / messageTotal // toUserId // this.editorType = config.editorType; this.editorMode = config.editorMode; - if(config.messageAreaName) { - this.messageAreaName = config.messageAreaName; + if(config.messageAreaTag) { + this.messageAreaTag = config.messageAreaTag; } this.messageIndex = config.messageIndex || 0; @@ -121,8 +123,8 @@ function FullScreenEditorModule(options) { // extraArgs can override some config if(_.isObject(options.extraArgs)) { - if(options.extraArgs.messageAreaName) { - this.messageAreaName = options.extraArgs.messageAreaName; + if(options.extraArgs.messageAreaTag) { + this.messageAreaTag = options.extraArgs.messageAreaTag; } if(options.extraArgs.messageIndex) { this.messageIndex = options.extraArgs.messageIndex; @@ -134,9 +136,6 @@ function FullScreenEditorModule(options) { this.toUserId = options.extraArgs.toUserId; } } - - console.log(this.toUserId) - console.log(this.messageAreaName) this.isReady = false; @@ -149,7 +148,7 @@ function FullScreenEditorModule(options) { }; this.isLocalEmail = function() { - return Message.WellKnownAreaNames.Private === self.messageAreaName; + return Message.WellKnownAreaTags.Private === self.messageAreaTag; }; this.isReply = function() { @@ -217,7 +216,7 @@ function FullScreenEditorModule(options) { var headerValues = self.viewControllers.header.getFormData().value; var msgOpts = { - areaName : self.messageAreaName, + areaTag : self.messageAreaTag, toUserName : headerValues.to, fromUserName : headerValues.from, subject : headerValues.subject, @@ -235,7 +234,7 @@ function FullScreenEditorModule(options) { self.message = message; updateMessageAreaLastReadId( - self.client.user.userId, self.messageAreaName, self.message.messageId, + self.client.user.userId, self.messageAreaTag, self.message.messageId, function lastReadUpdated() { if(self.isReady) { @@ -631,7 +630,7 @@ function FullScreenEditorModule(options) { }; this.initHeaderGeneric = function() { - self.setHeaderText(MCICodeIds.ViewModeHeader.AreaName, getMessageAreaByName(self.messageAreaName).desc); + self.setHeaderText(MCICodeIds.ViewModeHeader.AreaName, getMessageAreaByTag(self.messageAreaTag).name); }; this.initHeaderViewMode = function() { @@ -965,13 +964,10 @@ function FullScreenEditorModule(options) { require('util').inherits(FullScreenEditorModule, MenuModule); -FullScreenEditorModule.prototype.enter = function(client) { - FullScreenEditorModule.super_.prototype.enter.call(this, client); - - +FullScreenEditorModule.prototype.enter = function() { + FullScreenEditorModule.super_.prototype.enter.call(this); }; FullScreenEditorModule.prototype.mciReady = function(mciData, cb) { this.mciReadyHandler(mciData, cb); - //this['mciReadyHandler' + _.capitalize(this.editorType)](mciData); }; diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index a63ea79f..a9603f2e 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -1,9 +1,10 @@ /* jslint node: true */ 'use strict'; -var MailPacket = require('./mail_packet.js'); +//var MailPacket = require('./mail_packet.js'); var ftn = require('./ftn_util.js'); var Message = require('./message.js'); +var sauce = require('./sauce.js'); var _ = require('lodash'); var assert = require('assert'); @@ -12,6 +13,8 @@ var fs = require('fs'); var util = require('util'); var async = require('async'); var iconv = require('iconv-lite'); +var buffers = require('buffers'); +var moment = require('moment'); /* :TODO: should probably be broken up @@ -20,6 +23,493 @@ var iconv = require('iconv-lite'); FTNPacketExport: message(s) -> packet */ +/* +Reader: file to ftn data +Writer: ftn data to packet + +Data to toMessage +Data.fromMessage + +FTNMessage.toMessage() => Message +FTNMessage.fromMessage() => Create from Message + +* read: header -> simple {} obj, msg -> Message object +* read: read(..., iterator): iterator('header', ...), iterator('message', msg) +* write: provide information to go into header + +* Logic of "Is this for us"/etc. elsewhere +*/ + +const FTN_PACKET_HEADER_SIZE = 58; // fixed header size +const FTN_PACKET_HEADER_TYPE = 2; +const FTN_PACKET_MESSAGE_TYPE = 2; + +// EOF + SAUCE.id + SAUCE.version ('00') +const FTN_MESSAGE_SAUCE_HEADER = + new Buffer( [ 0x1a, 'S', 'A', 'U', 'C', 'E', '0', '0' ] ); + +const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; + +function FTNPacket() { + + var self = this; + + this.parsePacketHeader = function(packetBuffer, cb) { + assert(Buffer.isBuffer(packetBuffer)); + + // + // See the following specs: + // http://ftsc.org/docs/fts-0001.016 + // http://ftsc.org/docs/fsc-0048.002 + // + if(packetBuffer.length < FTN_PACKET_HEADER_SIZE) { + cb(new Error('Buffer too small')); + return; + } + + binary.parse(packetBuffer) + .word16lu('origNode') + .word16lu('destNode') + .word16lu('year') + .word16lu('month') + .word16lu('day') + .word16lu('hour') + .word16lu('minute') + .word16lu('second') + .word16lu('baud') + .word16lu('packetType') + .word16lu('origNet') + .word16lu('destNet') + .word8('prodCodeLo') + .word8('revisionMajor') // aka serialNo + .buffer('password', 8) // null padded C style string + .word16lu('origZone') + .word16lu('destZone') + // Additions in FSC-0048.002 follow... + .word16lu('auxNet') + .word16lu('capWordA') + .word8('prodCodeHi') + .word8('revisionMinor') + .word16lu('capWordB') + .word16lu('originZone2') + .word16lu('destZone2') + .word16lu('originPoint') + .word16lu('destPoint') + .word32lu('prodData') + .tap(packetHeader => { + // Convert password from NULL padded array to string + packetHeader.password = ftn.stringFromFTN(packetHeader.password); + + if(FTN_PACKET_HEADER_TYPE !== packetHeader.packetType) { + cb(new Error('Unsupported header type: ' + packetHeader.packetType)); + return; + } + + // + // Date/time components into something more reasonable + // Note: The names above match up with object members moment() allows + // + packetHeader.created = moment(packetHeader); + + cb(null, packetHeader); + }); + }; + + this.writePacketHeader = function(headerInfo, ws) { + let buffer = new Buffer(FTN_PACKET_HEADER_SIZE); + + buffer.writeUInt16LE(headerInfo.origNode, 0); + buffer.writeUInt16LE(headerInfo.destNode, 2); + buffer.writeUInt16LE(headerInfo.created.year(), 4); + buffer.writeUInt16LE(headerInfo.created.month(), 6); + buffer.writeUInt16LE(headerInfo.created.date(), 8); + buffer.writeUInt16LE(headerInfo.created.hour(), 10); + buffer.writeUInt16LE(headerInfo.created.minute(), 12); + buffer.writeUInt16LE(headerInfo.created.second(), 14); + buffer.writeUInt16LE(headerInfo.baud, 16); + buffer.writeUInt16LE(FTN_PACKET_HEADER_TYPE, 18); + buffer.writeUInt16LE(headerInfo.origNet, 20); + buffer.writeUInt16LE(headerInfo.destNet, 22); + buffer.writeUInt8(headerInfo.prodCodeLo, 24); + buffer.writeUInt8(headerInfo.revisionMajor, 25); + + const pass = ftn.stringToNullPaddedBuffer(headerInfo.password, 8); + pass.copy(buffer, 26); + + buffer.writeUInt16LE(headerInfo.origZone, 34); + buffer.writeUInt16LE(headerInfo.destZone, 36); + + // FSC-0048.002 additions... + buffer.writeUInt16LE(headerInfo.auxNet, 38); + buffer.writeUInt16LE(headerInfo.capWordA, 40); + buffer.writeUInt8(headerInfo.prodCodeHi, 42); + buffer.writeUInt8(headerInfo.revisionMinor, 43); + buffer.writeUInt16LE(headerInfo.capWordB, 44); + buffer.writeUInt16LE(headerInfo.origZone2, 46); + buffer.writeUInt16LE(headerInfo.destZone2, 48); + buffer.writeUInt16LE(headerInfo.origPoint, 50); + buffer.writeUInt16LE(headerInfo.destPoint, 52); + buffer.writeUInt32LE(headerInfo.prodData, 54); + + ws.write(buffer); + }; + + this.processMessageBody = function(messageBodyBuffer, cb) { + // + // From FTS-0001.16: + // "Message text is unbounded and null terminated (note exception below). + // + // A 'hard' carriage return, 0DH, marks the end of a paragraph, and must + // be preserved. + // + // So called 'soft' carriage returns, 8DH, may mark a previous + // processor's automatic line wrap, and should be ignored. Beware that + // they may be followed by linefeeds, or may not. + // + // All linefeeds, 0AH, should be ignored. Systems which display message + // text should wrap long lines to suit their application." + // + // This can be a bit tricky: + // * Decoding as CP437 converts 0x8d -> 0xec, so we'll need to correct for that + // * Many kludge lines specify an encoding. If we find one of such lines, we'll + // likely need to re-decode as the specified encoding + // * SAUCE is binary-ish data, so we need to inspect for it before any + // decoding occurs + // + let messageBodyData = { + message : [], + kludgeLines : {}, // KLUDGE:[value1, value2, ...] map + seenBy : [], + }; + + function addKludgeLine(line) { + const sepIndex = line.indexOf(':'); + const key = line.substr(0, sepIndex).toUpperCase(); + const value = line.substr(sepIndex + 1).trim(); + + // + // Allow mapped value to be either a key:value if there is only + // one entry, or key:[value1, value2,...] if there are more + // + if(messageBodyData.kludgeLines[key]) { + if(!_.isArray(messageBodyData.kludgeLines[key])) { + messageBodyData.kludgeLines[key] = [ messageBodyData.kludgeLines[key] ]; + } + messageBodyData.kludgeLines[key].push(value); + } else { + messageBodyData.kludgeLines[key] = value; + } + } + + async.series( + [ + function extractSauce(callback) { + // :TODO: This is wrong: SAUCE may not have EOF marker for one, also if it's + // present, we need to extract it but keep the rest of hte message intact as it likely + // has SEEN-BY, PATH, and other kludge information *appended* + const sauceHeaderPosition = messageBodyBuffer.indexOf(FTN_MESSAGE_SAUCE_HEADER); + if(sauceHeaderPosition > -1) { + sauce.readSAUCE(messageBodyBuffer.slice(sauceHeaderPosition), (err, theSauce) => { + if(!err) { + // we read some SAUCE - don't re-process that portion into the body + messageBodyBuffer = messageBodyBuffer.slice(0, sauceHeaderPosition); + messageBodyData.sauce = theSauce; + } + callback(null); // failure to read SAUCE is OK + }); + } else { + callback(null); + } + }, + function extractMessageData(callback) { + const messageLines = + iconv.decode(messageBodyBuffer, 'CP437').replace(/[\xec\n]/g, '').split(/\r/g); + + let preOrigin = true; + + messageLines.forEach(line => { + if(0 === line.length) { + messageBodyData.message.push(''); + return; + } + + if(preOrigin) { + if(line.startsWith('AREA:')) { + messageBodyData.area = line.substring(line.indexOf(':') + 1).trim(); + } else if(line.startsWith('--- ')) { + // Tear Lines are tracked allowing for specialized display/etc. + messageBodyData.tearLine = line; + } else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) { // To spec is " * Origin: ..." + messageBodyData.originLine = line; + preOrigin = false; + } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { + addKludgeLine(line.slice(1)); + } else { + // regular ol' message line + messageBodyData.message.push(line); + } + } else { + if(line.startsWith('SEEN-BY:')) { + messageBodyData.seenBy.push(line.substring(line.indexOf(':') + 1).trim()); + } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { + addKludgeLine(line.slice(1)); + } + } + }); + + callback(null); + } + ], + function complete(err) { + messageBodyData.message = messageBodyData.message.join('\n'); + cb(messageBodyData); + } + ); + }; + + this.parsePacketMessages = function(messagesBuffer, iterator, cb) { + const NULL_TERM_BUFFER = new Buffer( [ 0 ] ); + + binary.stream(messagesBuffer).loop(function looper(end, vars) { + // + // Some variable names used here match up directly with well known + // meta data names used with FTN messages. + // + this + .word16lu('messageType') + .word16lu('ftn_orig_node') + .word16lu('ftn_dest_node') + .word16lu('ftn_orig_network') + .word16lu('ftn_dest_network') + .word8('ftn_attr_flags1') + .word8('ftn_attr_flags2') + .word16lu('ftn_cost') + .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max + .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max + .scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max + .scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max + .scan('message', NULL_TERM_BUFFER) + .tap(function tapped(msgData) { + if(!msgData.ftn_orig_node) { + // end marker -- no more messages + end(); + cb(null); + return; + } + + if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) { + end(); + cb(new Error('Unsupported message type: ' + msgData.messageType)); + return; + } + + // + // Convert null terminated arrays to strings + // + let convMsgData = {}; + [ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => { + convMsgData[k] = iconv.decode(msgData[k], 'CP437'); + }); + + // + // The message body itself is a special beast as it may + // contain special origin lines, kludges, SAUCE in the case + // of ANSI files, etc. + // + let msg = new Message( { + toUserName : convMsgData.toUserName, + fromUserName : convMsgData.fromUserName, + subject : convMsgData.subject, + modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime), + }); + + msg.meta.FtnProperty = {}; + msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node; + msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node; + msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; + msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; + msg.meta.FtnProperty.ftn_attr_flags1 = msgData.ftn_attr_flags1; + msg.meta.FtnProperty.ftn_attr_flags2 = msgData.ftn_attr_flags2; + msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; + + self.processMessageBody(msgData.message, function processed(messageBodyData) { + msg.message = messageBodyData.message; + msg.meta.FtnKludge = messageBodyData.kludgeLines; + + if(messageBodyData.tearLine) { + msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine; + } + if(messageBodyData.seenBy.length > 0) { + msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy; + } + if(messageBodyData.area) { + msg.meta.FtnProperty.ftn_area = messageBodyData.area; + } + if(messageBodyData.originLine) { + msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine; + } + + iterator('message', msg); + }) + }); + }); + }; + + this.writeMessage = function(message, ws) { + let basicHeader = new Buffer(34); + + basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); + basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags1, 10); + basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags2, 11); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); + + // + // From http://ftsc.org/docs/fts-0001.016: + // DateTime = (* a character string 20 characters long *) + // (* 01 Jan 86 02:34:56 *) + // DayOfMonth " " Month " " Year " " + // " " HH ":" MM ":" SS + // Null + // + // DayOfMonth = "01" | "02" | "03" | ... | "31" (* Fido 0 fills *) + // Month = "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | + // "Jul" | "Aug" | "Sep" | "Oct" | "Nov" | "Dec" + // Year = "01" | "02" | .. | "85" | "86" | ... | "99" | "00" + // HH = "00" | .. | "23" + // MM = "00" | .. | "59" + // SS = "00" | .. | "59" + const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); + dateTimeBuffer.copy(basicHeader, 14); + + ws.write(basicHeader); + + // toUserName & fromUserName: up to 36 bytes in length, NULL term'd + // :TODO: DRY... + let encBuf = iconv.encode(message.toUserName + '\0', 'CP437').slice(0, 36); + encBuf[encBuf.length - 1] = '\0'; // ensure it's null term'd + ws.write(encBuf); + + encBuf = iconv.encode(message.fromUserName + '\0', 'CP437').slice(0, 36); + encBuf[encBuf.length - 1] = '\0'; // ensure it's null term'd + ws.write(encBuf); + + // subject: up to 72 bytes in length, NULL term'd + encBuf = iconv.encode(message.subject + '\0', 'CP437').slice(0, 72); + encBuf[encBuf.length - 1] = '\0'; // ensure it's null term'd + ws.write(encBuf); + + // + // message: unbound length, NULL term'd + // + // We need to build in various special lines - kludges, area, + // seen-by, etc. + // + // :TODO: Put this in it's own method + let msgBody = ''; + + function appendMeta(k, m) { + if(m) { + let a = m; + if(!_.isArray(a)) { + a = [ a ]; + } + a.forEach(v => { + msgBody += `${k}: ${v}\n`; + }); + } + } + + // :TODO: is Area really any differnt (e.g. no space between AREA:the_area) + if(message.meta.FtnProperty.ftn_area) { + msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\n`; + } + + Object.keys(message.meta.FtnKludge).forEach(k => { + if('PATH' !== k) { + appendMeta(k, message.meta.FtnKludge[k]); + } + }); + + msgBody += message.message; + + appendMeta('', message.meta.FtnProperty.ftn_tear_line); + appendMeta('', message.meta.FtnProperty.ftn_origin); + + appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); + appendMeta('PATH', message.meta.FtnKludge['PATH']); + + ws.write(iconv.encode(msgBody + '\0', 'CP437')); + }; + + this.parsePacketBuffer = function(packetBuffer, iterator, cb) { + async.series( + [ + function processHeader(callback) { + self.parsePacketHeader(packetBuffer, (err, header) => { + if(!err) { + iterator('header', header); + } + callback(err); + }); + }, + function processMessages(callback) { + self.parsePacketMessages( + packetBuffer.slice(FTN_PACKET_HEADER_SIZE), + iterator, + callback); + } + ], + cb + ); + }; +} + +FTNPacket.prototype.read = function(pathOrBuffer, iterator, cb) { + var self = this; + + async.series( + [ + function getBufferIfPath(callback) { + if(_.isString(pathOrBuffer)) { + fs.readFile(pathOrBuffer, (err, data) => { + pathOrBuffer = data; + callback(err); + }); + } else { + callback(null); + } + }, + function parseBuffer(callback) { + self.parsePacketBuffer(pathOrBuffer, iterator, callback); + } + ], + cb // completion callback + ); +}; + +FTNPacket.prototype.write = function(path, headerInfo, messages, cb) { + headerInfo.created = headerInfo.created || moment(); + headerInfo.baud = headerInfo.baud || 0; + // :TODO: Other defaults? + + if(!_.isArray(messages)) { + messages = [ messages ] ; + } + + let ws = fs.createWriteStream(path); + this.writePacketHeader(headerInfo, ws); + + messages.forEach(msg => { + this.writeMessage(msg, ws); + }); +}; + + + // // References // * http://ftsc.org/docs/fts-0001.016 @@ -30,7 +520,7 @@ var iconv = require('iconv-lite'); // function FTNMailPacket(options) { - MailPacket.call(this, options); + //MailPacket.call(this, options); var self = this; self.KLUDGE_PREFIX = '\x01'; @@ -77,7 +567,7 @@ function FTNMailPacket(options) { .word16lu('second') .word16lu('baud') .word16lu('packetType') - .word16lu('originNet') + .word16lu('origNet') .word16lu('destNet') .word8('prodCodeLo') .word8('revisionMajor') // aka serialNo @@ -100,35 +590,110 @@ function FTNMailPacket(options) { // :TODO: Don't hard code magic # here if(2 !== packetHeader.packetType) { + console.log(packetHeader.packetType) cb(new Error('Packet is not Type-2')); return; } + + // :TODO: convert date information -> .created + + packetHeader.created = moment(packetHeader); + /* + packetHeader.year, packetHeader.month, packetHeader.day, packetHeader.hour, + packetHeader.minute, packetHeader.second);*/ // :TODO: validate & pass error if failure cb(null, packetHeader); }); }; + + this.getPacketHeaderBuffer = function(packetHeader, options) { + options = options || {}; + + if(options.created) { + options.created = moment(options.created); // ensure we have a moment obj + } else { + options.created = moment(); + } + + let buffer = new Buffer(58); + + buffer.writeUInt16LE(packetHeader.origNode, 0); + buffer.writeUInt16LE(packetHeader.destNode, 2); + buffer.writeUInt16LE(options.created.year(), 4); + buffer.writeUInt16LE(options.created.month(), 6); + buffer.writeUInt16LE(options.created.date(), 8); + buffer.writeUInt16LE(options.created.hour(), 10); + buffer.writeUInt16LE(options.created.minute(), 12); + buffer.writeUInt16LE(options.created.second(), 14); + buffer.writeUInt16LE(0x0000, 16); + buffer.writeUInt16LE(0x0002, 18); + buffer.writeUInt16LE(packetHeader.origNet, 20); + buffer.writeUInt16LE(packetHeader.destNet, 22); + buffer.writeUInt8(packetHeader.prodCodeLo, 24); + buffer.writeUInt8(packetHeader.revisionMajor, 25); + + const pass = ftn.stringToNullPaddedBuffer(packetHeader.password, 8); + pass.copy(buffer, 26); + + buffer.writeUInt16LE(packetHeader.origZone, 34); + buffer.writeUInt16LE(packetHeader.destZone, 36); + + // FSC-0048.002 additions... + buffer.writeUInt16LE(packetHeader.auxNet, 38); + buffer.writeUInt16LE(packetHeader.capWordA, 40); + buffer.writeUInt8(packetHeader.prodCodeHi, 42); + buffer.writeUInt8(packetHeader.revisionMinor, 43); + buffer.writeUInt16LE(packetHeader.capWordB, 44); + buffer.writeUInt16LE(packetHeader.origZone2, 46); + buffer.writeUInt16LE(packetHeader.destZone2, 48); + buffer.writeUInt16LE(packetHeader.origPoint, 50); + buffer.writeUInt16LE(packetHeader.destPoint, 52); + buffer.writeUInt32LE(packetHeader.prodData, 54); + + return buffer; + }; + + self.setOrAppend = function(value, dst) { + if(dst) { + if(!_.isArray(dst)) { + dst = [ dst ]; + } + + dst.push(value); + } else { + dst = value; + } + } - self.getMessageMeta = function(msgBody) { + self.getMessageMeta = function(msgBody, msgData) { var meta = { FtnKludge : msgBody.kludgeLines, FtnProperty : {}, }; if(msgBody.tearLine) { - meta.FtnProperty.ftn_tear_line = [ msgBody.tearLine ]; + meta.FtnProperty.ftn_tear_line = msgBody.tearLine; } if(msgBody.seenBy.length > 0) { meta.FtnProperty.ftn_seen_by = msgBody.seenBy; } if(msgBody.area) { - meta.FtnProperty.ftn_area = [ msgBody.area ]; + meta.FtnProperty.ftn_area = msgBody.area; } if(msgBody.originLine) { - meta.FtnProperty.ftn_origin = [ msgBody.originLine ]; + meta.FtnProperty.ftn_origin = msgBody.originLine; } + meta.FtnProperty.ftn_orig_node = msgData.origNode; + meta.FtnProperty.ftn_dest_node = msgData.destNode; + meta.FtnProperty.ftn_orig_network = msgData.origNet; + meta.FtnProperty.ftn_dest_network = msgData.destNet; + meta.FtnProperty.ftn_attr_flags1 = msgData.attrFlags1; + meta.FtnProperty.ftn_attr_flags2 = msgData.attrFlags2; + meta.FtnProperty.ftn_cost = msgData.cost; + return meta; }; @@ -172,13 +737,15 @@ function FTNMailPacket(options) { var preOrigin = true; function addKludgeLine(kl) { - var kludgeParts = kl.split(':'); + const kludgeParts = kl.split(':'); kludgeParts[0] = kludgeParts[0].toUpperCase(); kludgeParts[1] = kludgeParts[1].trim(); - (msgBody.kludgeLines[kludgeParts[0]] = msgBody.kludgeLines[kludgeParts[0]] || []).push(kludgeParts[1]); + self.setOrAppend(kludgeParts[1], msgBody.kludgeLines[kludgeParts[0]]); } + var sauceBuffers; + msgLines.forEach(function nextLine(line) { if(0 === line.length) { msgBody.message.push(''); @@ -196,10 +763,12 @@ function FTNMailPacket(options) { preOrigin = false; } else if(self.KLUDGE_PREFIX === line.charAt(0)) { addKludgeLine(line.slice(1)); + } else if(!sauceBuffers || _.startsWith(line, '\x1aSAUCE00')) { + sauceBuffers = sauceBuffers || buffers(); + sauceBuffers.push(new Buffer(line)); } else { msgBody.message.push(line); } - // :TODO: SAUCE/etc. can be present? } else { if(_.startsWith(line, 'SEEN-BY:')) { msgBody.seenBy.push(line.substring(line.indexOf(':') + 1).trim()); @@ -209,29 +778,36 @@ function FTNMailPacket(options) { } }); + if(sauceBuffers) { + // :TODO: parse sauce -> sauce buffer. This needs changes to this method to return message & optional sauce + } + cb(null, msgBody); }; - this.extractMessages = function(buffer, cb) { - var nullTermBuf = new Buffer( [ 0 ] ); + this.extractMessages = function(buffer, iterator, cb) { + assert(Buffer.isBuffer(buffer)); + assert(_.isFunction(iterator)); + + const NULL_TERM_BUFFER = new Buffer( [ 0 ] ); binary.stream(buffer).loop(function looper(end, vars) { this .word16lu('messageType') - .word16lu('originNode') + .word16lu('origNode') .word16lu('destNode') - .word16lu('originNet') + .word16lu('origNet') .word16lu('destNet') .word8('attrFlags1') .word8('attrFlags2') .word16lu('cost') - .scan('modDateTime', nullTermBuf) - .scan('toUserName', nullTermBuf) - .scan('fromUserName', nullTermBuf) - .scan('subject', nullTermBuf) - .scan('message', nullTermBuf) + .scan('modDateTime', NULL_TERM_BUFFER) + .scan('toUserName', NULL_TERM_BUFFER) + .scan('fromUserName', NULL_TERM_BUFFER) + .scan('subject', NULL_TERM_BUFFER) + .scan('message', NULL_TERM_BUFFER) .tap(function tapped(msgData) { - if(!msgData.originNode) { + if(!msgData.origNode) { end(); cb(null); return; @@ -247,20 +823,25 @@ function FTNMailPacket(options) { // Now, create a Message object // var msg = new Message( { - // :TODO: areaId needs to be looked up via AREA line - may need a 1:n alias -> area ID lookup + // AREA FTN -> local conf/area occurs elsewhere toUserName : msgData.toUserName, fromUserName : msgData.fromUserName, subject : msgData.subject, message : msgBody.message.join('\n'), // :TODO: \r\n is better? modTimestamp : ftn.getDateFromFtnDateTime(msgData.modDateTime), - meta : self.getMessageMeta(msgBody), + meta : self.getMessageMeta(msgBody, msgData), + + }); - - self.emit('message', msg); // :TODO: Placeholder + + iterator(msg); + //self.emit('message', msg); // :TODO: Placeholder }); }); }); }; + + //this.getMessageHeaderBuffer = function(headerInfo) this.parseFtnMessages = function(buffer, cb) { var nullTermBuf = new Buffer( [ 0 ] ); @@ -269,9 +850,9 @@ function FTNMailPacket(options) { binary.stream(buffer).loop(function looper(end, vars) { this .word16lu('messageType') - .word16lu('originNode') + .word16lu('origNode') .word16lu('destNode') - .word16lu('originNet') + .word16lu('origNet') .word16lu('destNet') .word8('attrFlags1') .word8('attrFlags2') @@ -282,7 +863,7 @@ function FTNMailPacket(options) { .scan('subject', nullTermBuf) .scan('message', nullTermBuf) .tap(function tapped(msgData) { - if(!msgData.originNode) { + if(!msgData.origNode) { end(); cb(null, fidoMessages); return; @@ -302,7 +883,10 @@ function FTNMailPacket(options) { }); }; - this.extractMesssagesFromPacketBuffer = function(packetBuffer, cb) { + this.extractMesssagesFromPacketBuffer = function(packetBuffer, iterator, cb) { + assert(Buffer.isBuffer(packetBuffer)); + assert(_.isFunction(iterator)); + async.waterfall( [ function parseHeader(callback) { @@ -318,7 +902,8 @@ function FTNMailPacket(options) { }, function extractEmbeddedMessages(callback) { // note: packet header is 58 bytes in length - self.extractMessages(packetBuffer.slice(58), function extracted(err) { + self.extractMessages( + packetBuffer.slice(58), iterator, function extracted(err) { callback(err); }); } @@ -361,7 +946,7 @@ function FTNMailPacket(options) { }; } -require('util').inherits(FTNMailPacket, MailPacket); +//require('util').inherits(FTNMailPacket, MailPacket); FTNMailPacket.prototype.parse = function(path, cb) { var self = this; @@ -385,41 +970,67 @@ FTNMailPacket.prototype.parse = function(path, cb) { ); }; -FTNMailPacket.prototype.read = function(options) { - FTNMailPacket.super_.prototype.read.call(this, options); - +FTNMailPacket.prototype.read = function(pathOrBuffer, iterator, cb) { var self = this; - if(_.isString(options.packetPath)) { + if(_.isString(pathOrBuffer)) { async.waterfall( [ function readPacketFile(callback) { - fs.readFile(options.packetPath, function packetData(err, data) { + fs.readFile(pathOrBuffer, function packetData(err, data) { callback(err, data); }); }, function extractMessages(data, callback) { - self.extractMesssagesFromPacketBuffer(data, function extracted(err) { - callback(err); - }); + self.extractMesssagesFromPacketBuffer(data, iterator, callback); } ], - function complete(err) { - if(err) { - self.emit('error', err); - } - } + cb ); - } else if(Buffer.isBuffer(options.packetBuffer)) { + } else if(Buffer.isBuffer(pathOrBuffer)) { } }; -FTNMailPacket.prototype.write = function(options) { - FTNMailPacket.super_.prototype.write.call(this, options); +FTNMailPacket.prototype.write = function(messages, fileName, options) { + if(!_.isArray(messages)) { + messages = [ messages ]; + } + + + }; +var ftnPacket = new FTNPacket(); +var theHeader; +var written = false; +ftnPacket.read( + process.argv[2], + function iterator(dataType, data) { + if('header' === dataType) { + theHeader = data; + console.log(theHeader); + } else if('message' === dataType) { + const msg = data; + console.log(msg); + if(!written) { + written = true; + + let messages = [ msg ]; + ftnPacket.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messages, err => { + + }); + + } + } + }, + function completion(err) { + console.log(err); + } +); + +/* var mailPacket = new FTNMailPacket( { nodeAddresses : { @@ -434,11 +1045,42 @@ var mailPacket = new FTNMailPacket( } ); -mailPacket.on('message', function msgParsed(msg) { - console.log(msg); -}); -mailPacket.read( { packetPath : '/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007' } ); +var didWrite = false; +mailPacket.read( + process.argv[2], + //'/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/mf/extracted/27000425.pkt', + function packetIter(msg) { + console.log(msg); + if(_.has(msg, 'meta.FtnProperty.ftn_area')) { + console.log('AREA: ' + msg.meta.FtnProperty.ftn_area); + } + + if(!didWrite) { + console.log(mailPacket.packetHeader); + console.log('-----------'); + + + didWrite = true; + + let outTest = fs.createWriteStream('/home/nuskooler/Downloads/ftnout/test1.pkt'); + let buffer = mailPacket.getPacketHeaderBuffer(mailPacket.packetHeader); + //mailPacket.write(buffer, msg.packetHeader); + outTest.write(buffer); + } + }, + function complete(err) { + console.log(err); + } +); +*/ +/* + Area Map + networkName: { + area_tag: conf_name:area_tag_name + ... + } +*/ /* mailPacket.parse('/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007', function parsed(err, messages) { diff --git a/core/ftn_util.js b/core/ftn_util.js index 6ff0430b..41c8dfc6 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -10,11 +10,14 @@ var binary = require('binary'); var fs = require('fs'); var util = require('util'); var iconv = require('iconv-lite'); +var moment = require('moment'); // :TODO: Remove "Ftn" from most of these -- it's implied in the module exports.stringFromFTN = stringFromFTN; +exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; exports.getFormattedFTNAddress = getFormattedFTNAddress; exports.getDateFromFtnDateTime = getDateFromFtnDateTime; +exports.getDateTimeString = getDateTimeString; exports.getQuotePrefix = getQuotePrefix; @@ -33,6 +36,14 @@ function stringFromFTN(buf, encoding) { return iconv.decode(buf.slice(0, nullPos), encoding || 'utf-8'); } +function stringToNullPaddedBuffer(s, bufLen) { + let buffer = new Buffer(bufLen).fill(0x00); + let enc = iconv.encode(s, 'CP437').slice(0, bufLen); + for(let i = 0; i < enc.length; ++i) { + buffer[i] = enc[i]; + } + return buffer; +} // // Convert a FTN style DateTime string to a Date object @@ -44,9 +55,34 @@ function getDateFromFtnDateTime(dateTime) { // "Tue 01 Jan 80 00:00" // "27 Feb 15 00:00:03" // + // :TODO: Use moment.js here return (new Date(Date.parse(dateTime))).toISOString(); } +function getDateTimeString(m) { + // + // From http://ftsc.org/docs/fts-0001.016: + // DateTime = (* a character string 20 characters long *) + // (* 01 Jan 86 02:34:56 *) + // DayOfMonth " " Month " " Year " " + // " " HH ":" MM ":" SS + // Null + // + // DayOfMonth = "01" | "02" | "03" | ... | "31" (* Fido 0 fills *) + // Month = "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | + // "Jul" | "Aug" | "Sep" | "Oct" | "Nov" | "Dec" + // Year = "01" | "02" | .. | "85" | "86" | ... | "99" | "00" + // HH = "00" | .. | "23" + // MM = "00" | .. | "59" + // SS = "00" | .. | "59" + // + if(!moment.isMoment(m)) { + m = moment(m); + } + + return m.format('DD MMM YY HH:mm:ss'); +} + function getFormattedFTNAddress(address, dimensions) { //var addr = util.format('%d:%d', address.zone, address.net); var addr = '{0}:{1}'.format(address.zone, address.net); diff --git a/core/menu_module.js b/core/menu_module.js index e052b1db..503829c1 100644 --- a/core/menu_module.js +++ b/core/menu_module.js @@ -25,6 +25,10 @@ function MenuModule(options) { var self = this; this.menuName = options.menuName; this.menuConfig = options.menuConfig; + this.client = options.client; + + // :TODO: this and the line below with .config creates empty ({}) objects in the theme -- + // ...which we really should not do. If they aren't there already, don't use 'em. this.menuConfig.options = options.menuConfig.options || {}; this.menuMethods = {}; // methods called from @method's @@ -190,10 +194,7 @@ require('util').inherits(MenuModule, PluginModule); require('./mod_mixins.js').ViewControllerManagement.call(MenuModule.prototype); -MenuModule.prototype.enter = function(client) { - this.client = client; - assert(_.isObject(client)); - +MenuModule.prototype.enter = function() { if(_.isString(this.menuConfig.status)) { this.client.currentStatus = this.menuConfig.status; } else { diff --git a/core/menu_stack.js b/core/menu_stack.js index 5caf6531..64e8df40 100644 --- a/core/menu_stack.js +++ b/core/menu_stack.js @@ -131,7 +131,7 @@ MenuStack.prototype.goto = function(name, options, cb) { modInst.restoreSavedState(options.savedState); } - modInst.enter(self.client); + modInst.enter(); self.client.log.trace( { stack : _.map(self.stack, function(si) { return si.name; } ) }, diff --git a/core/menu_util.js b/core/menu_util.js index 601888a8..a00edb0b 100644 --- a/core/menu_util.js +++ b/core/menu_util.js @@ -4,10 +4,8 @@ // ENiGMA½ var moduleUtil = require('./module_util.js'); var Log = require('./logger.js').log; -var conf = require('./config.js'); // :TODO: remove me! var Config = require('./config.js').config; var asset = require('./asset.js'); -var theme = require('./theme.js'); var getFullConfig = require('./config_util.js').getFullConfig; var MCIViewFactory = require('./mci_view_factory.js').MCIViewFactory; var acsUtil = require('./acs_util.js'); @@ -68,17 +66,18 @@ function loadMenu(options, cb) { }); }, function loadMenuModule(menuConfig, callback) { - var modAsset = asset.getModuleAsset(menuConfig.module); - var modSupplied = null !== modAsset; - var modLoadOpts = { + const modAsset = asset.getModuleAsset(menuConfig.module); + const modSupplied = null !== modAsset; + + const modLoadOpts = { name : modSupplied ? modAsset.asset : 'standard_menu', path : (!modSupplied || 'systemModule' === modAsset.type) ? __dirname : Config.paths.mods, category : (!modSupplied || 'systemModule' === modAsset.type) ? null : 'mods', }; moduleUtil.loadModuleEx(modLoadOpts, function moduleLoaded(err, mod) { - var modData = { + const modData = { name : modLoadOpts.name, config : menuConfig, mod : mod, @@ -97,7 +96,8 @@ function loadMenu(options, cb) { { menuName : options.name, menuConfig : modData.config, - extraArgs : options.extraArgs + extraArgs : options.extraArgs, + client : options.client, }); callback(null, moduleInstance); } catch(e) { @@ -174,7 +174,7 @@ function handleAction(client, formData, conf) { assert(_.isObject(conf)); assert(_.isString(conf.action)); - var actionAsset = asset.parseAsset(conf.action); + const actionAsset = asset.parseAsset(conf.action); assert(_.isObject(actionAsset)); switch(actionAsset.type) { @@ -245,84 +245,3 @@ function handleNext(client, nextSpec, conf) { break; } } - - - -// :TODO: Seems better in theme.js, but that includes ViewController...which would then include theme.js -// ...theme.js only brings in VC to create themed pause prompt. Perhaps that should live elsewhere -/* -function applyGeneralThemeCustomization(options) { - // - // options.name - // options.client - // options.type - // options.config - // - assert(_.isString(options.name)); - assert(_.isObject(options.client)); - assert("menus" === options.type || "prompts" === options.type); - - if(_.has(options.client.currentTheme, [ 'customization', options.type, options.name ])) { - var themeConfig = options.client.currentTheme.customization[options.type][options.name]; - - if(themeConfig.config) { - Object.keys(themeConfig.config).forEach(function confEntry(conf) { - if(options.config[conf]) { - _.defaultsDeep(options.config[conf], themeConfig.config[conf]); - } else { - options.config[conf] = themeConfig.config[conf]; - } - }); - } - } -} -*/ - -/* -function applyMciThemeCustomization(options) { - // - // options.name : menu/prompt name - // options.mci : menu/prompt .mci section - // options.client : client - // options.type : menu|prompt - // options.formId : (optional) form ID in cases where multiple forms may exist wanting their own customization - // - // In the case of formId, the theme must include the ID as well, e.g.: - // { - // ... - // "2" : { - // "TL1" : { ... } - // } - // } - // - assert(_.isString(options.name)); - assert("menus" === options.type || "prompts" === options.type); - assert(_.isObject(options.client)); - - if(_.isUndefined(options.mci)) { - options.mci = {}; - } - - if(_.has(options.client.currentTheme, [ 'customization', options.type, options.name ])) { - var themeConfig = options.client.currentTheme.customization[options.type][options.name]; - - if(options.formId && _.has(themeConfig, options.formId.toString())) { - // form ID found - use exact match - themeConfig = themeConfig[options.formId]; - } - - if(themeConfig.mci) { - Object.keys(themeConfig.mci).forEach(function mciEntry(mci) { - // :TODO: a better way to do this? - if(options.mci[mci]) { - _.defaults(options.mci[mci], themeConfig.mci[mci]); - } else { - options.mci[mci] = themeConfig.mci[mci]; - } - }); - } - } - - // :TODO: apply generic stuff, e.g. "VM" (vs "VM1") -} -*/ \ No newline at end of file diff --git a/core/message.js b/core/message.js index 4b99fcfc..cdbfe67f 100644 --- a/core/message.js +++ b/core/message.js @@ -16,7 +16,7 @@ function Message(options) { options = options || {}; this.messageId = options.messageId || 0; // always generated @ persist - this.areaName = options.areaName || Message.WellKnownAreaNames.Invalid; + this.areaTag = options.areaTag || Message.WellKnownAreaTags.Invalid; this.uuid = uuid.v1(); this.replyToMsgId = options.replyToMsgId || 0; this.toUserName = options.toUserName || ''; @@ -55,7 +55,7 @@ function Message(options) { }; this.isPrivate = function() { - return this.areaName === Message.WellKnownAreaNames.Private ? true : false; + return this.areaTag === Message.WellKnownAreaTags.Private ? true : false; }; this.getMessageTimestampString = function(ts) { @@ -80,7 +80,7 @@ function Message(options) { */ } -Message.WellKnownAreaNames = { +Message.WellKnownAreaTags = { Invalid : '', Private : 'private_mail', Bulletin : 'local_bulletin', @@ -104,16 +104,21 @@ Message.SystemMetaNames = { LocalFromUserID : 'local_from_user_id', }; -Message.FtnPropertyNames = { - FtnCost : 'ftn_cost', +Message.FtnPropertyNames = { FtnOrigNode : 'ftn_orig_node', FtnDestNode : 'ftn_dest_node', FtnOrigNetwork : 'ftn_orig_network', FtnDestNetwork : 'ftn_dest_network', + FtnAttrFlags1 : 'ftn_attr_flags1', + FtnAttrFlags2 : 'ftn_attr_flags2', + FtnCost : 'ftn_cost', FtnOrigZone : 'ftn_orig_zone', FtnDestZone : 'ftn_dest_zone', FtnOrigPoint : 'ftn_orig_point', FtnDestPoint : 'ftn_dest_point', + + + FtnAttribute : 'ftn_attribute', FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001 @@ -141,7 +146,7 @@ Message.prototype.load = function(options, cb) { [ function loadMessage(callback) { msgDb.get( - 'SELECT message_id, area_name, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, ' + + 'SELECT message_id, area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, ' + 'message, modified_timestamp, view_count ' + 'FROM message ' + 'WHERE message_uuid=? ' + @@ -149,7 +154,7 @@ Message.prototype.load = function(options, cb) { [ options.uuid ], function row(err, msgRow) { self.messageId = msgRow.message_id; - self.areaName = msgRow.area_name; + self.areaTag = msgRow.area_tag; self.messageUuid = msgRow.message_uuid; self.replyToMsgId = msgRow.reply_to_message_id; self.toUserName = msgRow.to_user_name; @@ -202,8 +207,8 @@ Message.prototype.persist = function(cb) { }, function storeMessage(callback) { msgDb.run( - 'INSERT INTO message (area_name, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaName, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], + 'INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' + + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], function msgInsert(err) { if(!err) { self.messageId = this.lastID; diff --git a/core/message_area.js b/core/message_area.js index 53aec52b..957a0e37 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -5,100 +5,275 @@ var msgDb = require('./database.js').dbs.message; var Config = require('./config.js').config; var Message = require('./message.js'); var Log = require('./logger.js').log; +var checkAcs = require('./acs_util.js').checkAcs; var async = require('async'); var _ = require('lodash'); var assert = require('assert'); -exports.getAvailableMessageAreas = getAvailableMessageAreas; -exports.getDefaultMessageArea = getDefaultMessageArea; -exports.getMessageAreaByName = getMessageAreaByName; +exports.getAvailableMessageConferences = getAvailableMessageConferences; +exports.getSortedAvailMessageConferences = getSortedAvailMessageConferences; +exports.getAvailableMessageAreasByConfTag = getAvailableMessageAreasByConfTag; +exports.getSortedAvailMessageAreasByConfTag = getSortedAvailMessageAreasByConfTag; +exports.getDefaultMessageConferenceTag = getDefaultMessageConferenceTag; +exports.getDefaultMessageAreaTagByConfTag = getDefaultMessageAreaTagByConfTag; +exports.getMessageConferenceByTag = getMessageConferenceByTag; +exports.getMessageAreaByTag = getMessageAreaByTag; +exports.changeMessageConference = changeMessageConference; exports.changeMessageArea = changeMessageArea; exports.getMessageListForArea = getMessageListForArea; exports.getNewMessagesInAreaForUser = getNewMessagesInAreaForUser; exports.getMessageAreaLastReadId = getMessageAreaLastReadId; exports.updateMessageAreaLastReadId = updateMessageAreaLastReadId; -function getAvailableMessageAreas(options) { - // example: [ { "name" : "local_music", "desc" : "Music Discussion", "groups" : ["somegroup"] }, ... ] - options = options || {}; +const CONF_AREA_RW_ACS_DEFAULT = 'GM[users]'; +const AREA_MANAGE_ACS_DEFAULT = 'GM[sysops]'; - var areas = Config.messages.areas; - var avail = []; - for(var i = 0; i < areas.length; ++i) { - if(true !== options.includePrivate && - Message.WellKnownAreaNames.Private === areas[i].name) - { - continue; - } +const AREA_ACS_DEFAULT = { + read : CONF_AREA_RW_ACS_DEFAULT, + write : CONF_AREA_RW_ACS_DEFAULT, + manage : AREA_MANAGE_ACS_DEFAULT, +}; - avail.push(areas[i]); - } - - return avail; +function getAvailableMessageConferences(client, options) { + options = options || { includeSystemInternal : false }; + + // perform ACS check per conf & omit system_internal if desired + return _.omit(Config.messageConferences, (v, k) => { + if(!options.includeSystemInternal && 'system_internal' === k) { + return true; + } + + const readAcs = v.acs || CONF_AREA_RW_ACS_DEFAULT; + return !checkAcs(client, readAcs); + }); } -function getDefaultMessageArea() { - // - // Return first non-private/etc. area name. This will be from config.hjson - // - return getAvailableMessageAreas()[0]; - /* - var avail = getAvailableMessageAreas(); - for(var i = 0; i < avail.length; ++i) { - if(Message.WellKnownAreaNames.Private !== avail[i].name) { - return avail[i]; - } - } - */ -} - -function getMessageAreaByName(areaName) { - areaName = areaName.toLowerCase(); - - var availAreas = getAvailableMessageAreas( { includePrivate : true } ); - var index = _.findIndex(availAreas, function pred(an) { - return an.name == areaName; +function getSortedAvailMessageConferences(client, options) { + var sorted = _.map(getAvailableMessageConferences(client, options), (v, k) => { + return { + confTag : k, + conf : v, + }; + }); + + sorted.sort((a, b) => { + return a.conf.name.localeCompare(b.conf.name); }); - if(index > -1) { - return availAreas[index]; - } + return sorted; } -function changeMessageArea(client, areaName, cb) { +// Return an *object* of available areas within |confTag| +function getAvailableMessageAreasByConfTag(confTag, options) { + options = options || {}; + + if(_.has(Config.messageConferences, [ confTag, 'areas' ])) { + const areas = Config.messageConferences[confTag].areas; + + if(!options.client || true === options.noAcsCheck) { + // everything - no ACS checks + return areas; + } else { + // perform ACS check per area + return _.omit(areas, (v, k) => { + const readAcs = _.has(v, 'acs.read') ? v.acs.read : CONF_AREA_RW_ACS_DEFAULT; + return !checkAcs(options.client, readAcs); + }); + } + } +} + +function getSortedAvailMessageAreasByConfTag(confTag, options) { + const areas = getAvailableMessageAreasByConfTag(confTag, options); + + // :TODO: should probably be using localeCompare / sort + return _.sortBy(_.map(areas, (v, k) => { + return { + areaTag : k, + area : v, + }; + }), o => o.area.name); // sort by name +} + +function getDefaultMessageConferenceTag(client, disableAcsCheck) { + // + // Find the first conference marked 'default'. If found, + // inspect |client| against *read* ACS using defaults if not + // specified. + // + // If the above fails, just go down the list until we get one + // that passes. + // + // It's possible that we end up with nothing here! + // + // Note that built in 'system_internal' is always ommited here + // + let defaultConf = _.findKey(Config.messageConferences, o => o.default); + if(defaultConf) { + const acs = Config.messageConferences[defaultConf].acs || CONF_AREA_RW_ACS_DEFAULT; + if(true === disableAcsCheck || checkAcs(client, acs)) { + return defaultConf; + } + } + + // just use anything we can + defaultConf = _.findKey(Config.messageConferences, (o, k) => { + const acs = o.acs || CONF_AREA_RW_ACS_DEFAULT; + return 'system_internal' !== k && (true === disableAcsCheck || checkAcs(client, acs)); + }); + + return defaultConf; +} + +function getDefaultMessageAreaTagByConfTag(client, confTag, disableAcsCheck) { + // + // Similar to finding the default conference: + // Find the first entry marked 'default', if any. If found, check | client| against + // *read* ACS. If this fails, just find the first one we can that passes checks. + // + // It's possible that we end up with nothing! + // + confTag = confTag || getDefaultMessageConferenceTag(client); + + if(confTag && _.has(Config.messageConferences, [ confTag, 'areas' ])) { + const areaPool = Config.messageConferences[confTag].areas; + let defaultArea = _.findKey(areaPool, o => o.default); + if(defaultArea) { + const readAcs = _.has(areaPool, [ defaultArea, 'acs', 'read' ]) ? areaPool[defaultArea].acs.read : AREA_ACS_DEFAULT.read; + if(true === disableAcsCheck || checkAcs(client, readAcs)) { + return defaultArea; + } + } + + defaultArea = _.findKey(areaPool, (o, k) => { + const readAcs = _.has(areaPool, [ defaultArea, 'acs', 'read' ]) ? areaPool[defaultArea].acs.read : AREA_ACS_DEFAULT.read; + return (true === disableAcsCheck || checkAcs(client, readAcs)); + }); + + return defaultArea; + } +} + +function getMessageConferenceByTag(confTag) { + return Config.messageConferences[confTag]; +} + +function getMessageAreaByTag(areaTag, optionalConfTag) { + const confs = Config.messageConferences; + + if(_.isString(optionalConfTag)) { + if(_.has(confs, [ optionalConfTag, 'areas', areaTag ])) { + return confs[optionalConfTag].areas[areaTag]; + } + } else { + // + // No confTag to work with - we'll have to search through them all + // + var area; + _.forEach(confs, (v, k) => { + if(_.has(v, [ 'areas', areaTag ])) { + area = v.areas[areaTag]; + return false; // stop iteration + } + }); + + return area; + } +} + +function changeMessageConference(client, confTag, cb) { + async.waterfall( + [ + function getConf(callback) { + const conf = getMessageConferenceByTag(confTag); + + if(conf) { + callback(null, conf); + } else { + callback(new Error('Invalid message conference tag')); + } + }, + function getDefaultAreaInConf(conf, callback) { + const areaTag = getDefaultMessageAreaTagByConfTag(client, confTag); + const area = getMessageAreaByTag(areaTag, confTag); + + if(area) { + callback(null, conf, { areaTag : areaTag, area : area } ); + } else { + callback(new Error('No available areas for this user in conference')); + } + }, + function validateAccess(conf, areaInfo, callback) { + const confAcs = conf.acs || CONF_AREA_RW_ACS_DEFAULT; + + if(!checkAcs(client, confAcs)) { + callback(new Error('User does not have access to this conference')); + } else { + const areaAcs = _.has(areaInfo, 'area.acs.read') ? areaInfo.area.acs.read : CONF_AREA_RW_ACS_DEFAULT; + if(!checkAcs(client, areaAcs)) { + callback(new Error('User does not have access to default area in this conference')); + } else { + callback(null, conf, areaInfo); + } + } + }, + function changeConferenceAndArea(conf, areaInfo, callback) { + const newProps = { + message_conf_tag : confTag, + message_area_tag : areaInfo.areaTag, + }; + client.user.persistProperties(newProps, err => { + callback(err, conf, areaInfo); + }); + }, + ], + function complete(err, conf, areaInfo) { + if(!err) { + client.log.info( { confTag : confTag, confName : conf.name, areaTag : areaInfo.areaTag }, 'Current message conference changed'); + } else { + client.log.warn( { confTag : confTag, error : err.message }, 'Could not change message conference'); + } + cb(err); + } + ); +} + +function changeMessageArea(client, areaTag, cb) { async.waterfall( [ function getArea(callback) { - var area = getMessageAreaByName(areaName); + const area = getMessageAreaByTag(areaTag); if(area) { callback(null, area); } else { - callback(new Error('Invalid message area')); + callback(new Error('Invalid message area tag')); } }, function validateAccess(area, callback) { - if(_.isArray(area.groups) && ! - client.user.isGroupMember(area.groups)) - { + // + // Need at least *read* to access the area + // + const readAcs = _.has(area, 'acs.read') ? area.acs.read : CONF_AREA_RW_ACS_DEFAULT; + if(!checkAcs(client, readAcs)) { callback(new Error('User does not have access to this area')); } else { callback(null, area); } }, function changeArea(area, callback) { - client.user.persistProperty('message_area_name', area.name, function persisted(err) { + client.user.persistProperty('message_area_tag', areaTag, function persisted(err) { callback(err, area); }); } ], function complete(err, area) { if(!err) { - client.log.info( area, 'Current message area changed'); + client.log.info( { areaTag : areaTag, area : area }, 'Current message area changed'); } else { - client.log.warn( { area : area, error : err.message }, 'Could not change message area'); + client.log.warn( { areaTag : areaTag, area : area, error : err.message }, 'Could not change message area'); } cb(err); @@ -119,9 +294,9 @@ function getMessageFromRow(row) { }; } -function getNewMessagesInAreaForUser(userId, areaName, cb) { +function getNewMessagesInAreaForUser(userId, areaTag, cb) { // - // If |areaName| is Message.WellKnownAreaNames.Private, + // If |areaTag| is Message.WellKnownAreaTags.Private, // only messages addressed to |userId| should be returned. // // Only messages > lastMessageId should be returned @@ -131,7 +306,7 @@ function getNewMessagesInAreaForUser(userId, areaName, cb) { async.waterfall( [ function getLastMessageId(callback) { - getMessageAreaLastReadId(userId, areaName, function fetched(err, lastMessageId) { + getMessageAreaLastReadId(userId, areaTag, function fetched(err, lastMessageId) { callback(null, lastMessageId || 0); // note: willingly ignoring any errors here! }); }, @@ -139,9 +314,9 @@ function getNewMessagesInAreaForUser(userId, areaName, cb) { var sql = 'SELECT message_id, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, modified_timestamp, view_count ' + 'FROM message ' + - 'WHERE area_name="' + areaName + '" AND message_id > ' + lastMessageId; + 'WHERE area_tag ="' + areaTag + '" AND message_id > ' + lastMessageId; - if(Message.WellKnownAreaNames.Private === areaName) { + if(Message.WellKnownAreaTags.Private === areaTag) { sql += ' AND message_id in (' + 'SELECT message_id from message_meta where meta_category=' + Message.MetaCategories.System + @@ -150,8 +325,6 @@ function getNewMessagesInAreaForUser(userId, areaName, cb) { sql += ' ORDER BY message_id;'; - console.log(sql) - msgDb.each(sql, function msgRow(err, row) { if(!err) { msgList.push(getMessageFromRow(row)); @@ -160,18 +333,17 @@ function getNewMessagesInAreaForUser(userId, areaName, cb) { } ], function complete(err) { - console.log(msgList) cb(err, msgList); } ); } -function getMessageListForArea(options, areaName, cb) { +function getMessageListForArea(options, areaTag, cb) { // // options.client (required) // - options.client.log.debug( { areaName : areaName }, 'Fetching available messages'); + options.client.log.debug( { areaTag : areaTag }, 'Fetching available messages'); assert(_.isObject(options.client)); @@ -193,9 +365,9 @@ function getMessageListForArea(options, areaName, cb) { msgDb.each( 'SELECT message_id, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, modified_timestamp, view_count ' + 'FROM message ' + - 'WHERE area_name=? ' + + 'WHERE area_tag = ? ' + 'ORDER BY message_id;', - [ areaName.toLowerCase() ], + [ areaTag.toLowerCase() ], function msgRow(err, row) { if(!err) { msgList.push(getMessageFromRow(row)); @@ -214,24 +386,24 @@ function getMessageListForArea(options, areaName, cb) { ); } -function getMessageAreaLastReadId(userId, areaName, cb) { +function getMessageAreaLastReadId(userId, areaTag, cb) { msgDb.get( 'SELECT message_id ' + 'FROM user_message_area_last_read ' + - 'WHERE user_id = ? AND area_name = ?;', - [ userId, areaName ], + 'WHERE user_id = ? AND area_tag = ?;', + [ userId, areaTag ], function complete(err, row) { cb(err, row ? row.message_id : 0); } ); } -function updateMessageAreaLastReadId(userId, areaName, messageId, cb) { +function updateMessageAreaLastReadId(userId, areaTag, messageId, cb) { // :TODO: likely a better way to do this... async.waterfall( [ function getCurrent(callback) { - getMessageAreaLastReadId(userId, areaName, function result(err, lastId) { + getMessageAreaLastReadId(userId, areaTag, function result(err, lastId) { lastId = lastId || 0; callback(null, lastId); // ignore errors as we default to 0 }); @@ -239,25 +411,29 @@ function updateMessageAreaLastReadId(userId, areaName, messageId, cb) { function update(lastId, callback) { if(messageId > lastId) { msgDb.run( - 'REPLACE INTO user_message_area_last_read (user_id, area_name, message_id) ' + + 'REPLACE INTO user_message_area_last_read (user_id, area_tag, message_id) ' + 'VALUES (?, ?, ?);', - [ userId, areaName, messageId ], - callback + [ userId, areaTag, messageId ], + function written(err) { + callback(err, true); // true=didUpdate + } ); } else { callback(null); } } ], - function complete(err) { + function complete(err, didUpdate) { if(err) { Log.debug( - { error : err.toString(), userId : userId, areaName : areaName, messageId : messageId }, + { error : err.toString(), userId : userId, areaTag : areaTag, messageId : messageId }, 'Failed updating area last read ID'); } else { - Log.trace( - { userId : userId, areaName : areaName, messageId : messageId }, - 'Area last read ID updated'); + if(true === didUpdate) { + Log.trace( + { userId : userId, areaTag : areaTag, messageId : messageId }, + 'Area last read ID updated'); + } } cb(err); } diff --git a/core/module_util.js b/core/module_util.js index 55dd61ed..c66155f1 100644 --- a/core/module_util.js +++ b/core/module_util.js @@ -14,38 +14,40 @@ exports.loadModuleEx = loadModuleEx; exports.loadModule = loadModule; exports.loadModulesForCategory = loadModulesForCategory; + function loadModuleEx(options, cb) { assert(_.isObject(options)); assert(_.isString(options.name)); assert(_.isString(options.path)); - var modConfig = _.isObject(Config[options.category]) ? Config[options.category][options.name] : null; + const modConfig = _.isObject(Config[options.category]) ? Config[options.category][options.name] : null; if(_.isObject(modConfig) && false === modConfig.enabled) { cb(new Error('Module "' + options.name + '" is disabled')); return; } + var mod; try { - var mod = require(paths.join(options.path, options.name + '.js')); - - if(!_.isObject(mod.moduleInfo)) { - cb(new Error('Module is missing "moduleInfo" section')); - return; - } - - if(!_.isFunction(mod.getModule)) { - cb(new Error('Invalid or missing "getModule" method for module!')); - return; - } - - // Safe configuration, if any, for convience to the module - mod.runtime = { config : modConfig }; - - cb(null, mod); + mod = require(paths.join(options.path, options.name + '.js')); } catch(e) { cb(e); } + + if(!_.isObject(mod.moduleInfo)) { + cb(new Error('Module is missing "moduleInfo" section')); + return; + } + + if(!_.isFunction(mod.getModule)) { + cb(new Error('Invalid or missing "getModule" method for module!')); + return; + } + + // Ref configuration, if any, for convience to the module + mod.runtime = { config : modConfig }; + + cb(null, mod); } function loadModule(name, category, cb) { @@ -61,7 +63,7 @@ function loadModule(name, category, cb) { }); } -function loadModulesForCategory(category, cb) { +function loadModulesForCategory(category, iterator) { var path = Config.paths[category]; fs.readdir(path, function onFiles(err, files) { @@ -72,8 +74,7 @@ function loadModulesForCategory(category, cb) { var filtered = files.filter(function onFilter(file) { return '.js' === paths.extname(file); }); filtered.forEach(function onFile(file) { - var modName = paths.basename(file, '.js'); - loadModule(paths.basename(file, '.js'), category, cb); + loadModule(paths.basename(file, '.js'), category, iterator); }); }); } diff --git a/core/msg_network_module.js b/core/msg_network_module.js new file mode 100644 index 00000000..8d4a9e0c --- /dev/null +++ b/core/msg_network_module.js @@ -0,0 +1,25 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +var PluginModule = require('./plugin_module.js').PluginModule; + +exports.MessageNetworkModule = MessageNetworkModule; + +function MessageNetworkModule() { + PluginModule.call(this); +} + +require('util').inherits(MessageNetworkModule, PluginModule); + +MessageNetworkModule.prototype.startup = function(cb) { + cb(null); +}; + +MessageNetworkModule.prototype.shutdown = function(cb) { + cb(null); +}; + +MessageNetworkModule.prototype.record = function(message, cb) { + cb(null); +}; \ No newline at end of file diff --git a/core/msg_networks/ftn_msg_network_module.js b/core/msg_networks/ftn_msg_network_module.js new file mode 100644 index 00000000..f81aef9f --- /dev/null +++ b/core/msg_networks/ftn_msg_network_module.js @@ -0,0 +1,27 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +var MessageNetworkModule = require('./msg_network_module.js').MessageNetworkModule; + +function FTNMessageNetworkModule() { + MessageNetworkModule.call(this); +} + +require('util').inherits(FTNMessageNetworkModule, MessageNetworkModule); + +FTNMessageNetworkModule.prototype.startup = function(cb) { + cb(null); +}; + +FTNMessageNetworkModule.prototype.shutdown = function(cb) { + cb(null); +}; + +FTNMessageNetworkModule.prototype.record = function(message, cb) { + cb(null); + + // :TODO: should perhaps record in batches - e.g. start an event, record + // to temp location until time is hit or N achieved such that if multiple + // messages are being created a .FTN file is not made for each one +}; \ No newline at end of file diff --git a/core/new_scan.js b/core/new_scan.js index 3201402a..2b1f8a1f 100644 --- a/core/new_scan.js +++ b/core/new_scan.js @@ -7,6 +7,7 @@ var Message = require('./message.js'); var MenuModule = require('./menu_module.js').MenuModule; var ViewController = require('../core/view_controller.js').ViewController; +var _ = require('lodash'); var async = require('async'); exports.moduleInfo = { @@ -36,10 +37,11 @@ function NewScanModule(options) { var self = this; var config = this.menuConfig.config; - this.currentStep = 'messageAreas'; - this.currentScanAux = 0; // e.g. Message.WellKnownAreaNames.Private when currentSteps = messageAreas + this.currentStep = 'messageConferences'; + this.currentScanAux = {}; - this.scanStartFmt = config.scanStartFmt || 'Scanning {desc}...'; + // :TODO: Make this conf/area specific: + this.scanStartFmt = config.scanStartFmt || 'Scanning {confName} - {areaName}...'; this.scanFinishNoneFmt = config.scanFinishNoneFmt || 'Nothing new'; this.scanFinishNewFmt = config.scanFinishNewFmt || '{count} entries found'; this.scanCompleteMsg = config.scanCompleteMsg || 'Finished newscan'; @@ -57,10 +59,65 @@ function NewScanModule(options) { if(view) { } }; + + this.newScanMessageConference = function(cb) { + // lazy init + if(!self.sortedMessageConfs) { + const getAvailOpts = { includeSystemInternal : true }; // find new private messages, bulletins, etc. - this.newScanMessageArea = function(cb) { - var availMsgAreas = msgArea.getAvailableMessageAreas( { includePrivate : true } ); - var currentArea = availMsgAreas[self.currentScanAux]; + self.sortedMessageConfs = _.map(msgArea.getAvailableMessageConferences(self.client, getAvailOpts), (v, k) => { + return { + confTag : k, + conf : v, + }; + }); + + // + // Sort conferences by name, other than 'system_internal' which should + // always come first such that we display private mails/etc. before + // other conferences & areas + // + self.sortedMessageConfs.sort((a, b) => { + if('system_internal' === a.confTag) { + return -1; + } else { + return a.conf.name.localeCompare(b.conf.name); + } + }); + + self.currentScanAux.conf = self.currentScanAux.conf || 0; + self.currentScanAux.area = self.currentScanAux.area || 0; + } + + const currentConf = self.sortedMessageConfs[self.currentScanAux.conf]; + + async.series( + [ + function scanArea(callback) { + //self.currentScanAux.area = self.currentScanAux.area || 0; + + self.newScanMessageArea(currentConf, function areaScanComplete(err) { + if(self.sortedMessageConfs.length > self.currentScanAux.conf + 1) { + self.currentScanAux.conf += 1; + self.currentScanAux.area = 0; + + self.newScanMessageConference(cb); // recursive to next conf + //callback(null); + } else { + self.updateScanStatus(self.scanCompleteMsg); + callback(new Error('No more conferences')); + } + }); + } + ], + cb + ); + }; + + this.newScanMessageArea = function(conf, cb) { + // :TODO: it would be nice to cache this - must be done by conf! + const sortedAreas = msgArea.getSortedAvailMessageAreasByConfTag(conf.confTag, { client : self.client } ); + const currentArea = sortedAreas[self.currentScanAux.area]; // // Scan and update index until we find something. If results are found, @@ -70,8 +127,8 @@ function NewScanModule(options) { [ function checkAndUpdateIndex(callback) { // Advance to next area if possible - if(availMsgAreas.length >= self.currentScanAux + 1) { - self.currentScanAux += 1; + if(sortedAreas.length >= self.currentScanAux.area + 1) { + self.currentScanAux.area += 1; callback(null); } else { self.updateScanStatus(self.scanCompleteMsg); @@ -80,22 +137,30 @@ function NewScanModule(options) { }, function updateStatusScanStarted(callback) { self.updateScanStatus(self.scanStartFmt.format({ - desc : currentArea.desc, + confName : conf.conf.name, + confDesc : conf.conf.desc, + areaName : currentArea.area.name, + areaDesc : currentArea.area.desc, })); callback(null); }, function newScanAreaAndGetMessages(callback) { msgArea.getNewMessagesInAreaForUser( - self.client.user.userId, currentArea.name, function msgs(err, msgList) { + self.client.user.userId, currentArea.areaTag, function msgs(err, msgList) { if(!err) { if(0 === msgList.length) { self.updateScanStatus(self.scanFinishNoneFmt.format({ - desc : currentArea.desc, + confName : conf.conf.name, + confDesc : conf.conf.desc, + areaName : currentArea.area.name, + areaDesc : currentArea.area.desc, })); } else { self.updateScanStatus(self.scanFinishNewFmt.format({ - desc : currentArea.desc, - count : msgList.length, + confName : conf.conf.name, + confDesc : conf.conf.desc, + areaName : currentArea.area.name, + count : msgList.length, })); } } @@ -107,14 +172,14 @@ function NewScanModule(options) { if(msgList && msgList.length > 0) { var nextModuleOpts = { extraArgs: { - messageAreaName : currentArea.name, + messageAreaTag : currentArea.areaTag, messageList : msgList, } }; self.gotoMenu(config.newScanMessageList || 'newScanMessageList', nextModuleOpts); } else { - self.newScanMessageArea(cb); + self.newScanMessageArea(conf, cb); } } ], @@ -161,10 +226,10 @@ NewScanModule.prototype.mciReady = function(mciData, cb) { }, function performCurrentStepScan(callback) { switch(self.currentStep) { - case 'messageAreas' : - self.newScanMessageArea(function scanComplete(err) { - callback(null); // finished - }); + case 'messageConferences' : + self.newScanMessageConference(function scanComplete(err) { + callback(null); // finished + }); break; default : @@ -180,9 +245,3 @@ NewScanModule.prototype.mciReady = function(mciData, cb) { } ); }; - -/* -NewScanModule.prototype.finishedLoading = function() { - NewScanModule.super_.prototype.finishedLoading.call(this); -}; -*/ \ No newline at end of file diff --git a/core/predefined_mci.js b/core/predefined_mci.js index 2d9d4a7d..b8d7bbbc 100644 --- a/core/predefined_mci.js +++ b/core/predefined_mci.js @@ -3,7 +3,7 @@ var Config = require('./config.js').config; var Log = require('./logger.js').log; -var getMessageAreaByName = require('./message_area.js').getMessageAreaByName; +var getMessageAreaByTag = require('./message_area.js').getMessageAreaByTag; var clientConnections = require('./client_connections.js'); var sysProp = require('./system_property.js'); @@ -63,10 +63,15 @@ function getPredefinedMCIValue(client, code) { return _.has(client, 'currentMenuModule.menuConfig.desc') ? client.currentMenuModule.menuConfig.desc : ''; }, - MA : function messageAreaDescription() { - var area = getMessageAreaByName(client.user.properties.message_area_name); - return area ? area.desc : ''; + MA : function messageAreaName() { + const area = getMessageAreaByTag(client.user.properties.message_area_tag); + return area ? area.name : ''; }, + + ML : function messageAreaDescription() { + const area = getMessageAreaByTag(client.user.properties.message_area_tag); + return area ? area.desc : ''; + }, SH : function termHeight() { return client.term.termHeight.toString(); }, SW : function termWidth() { return client.term.termWidth.toString(); }, diff --git a/core/sauce.js b/core/sauce.js new file mode 100644 index 00000000..6db85533 --- /dev/null +++ b/core/sauce.js @@ -0,0 +1,165 @@ +/* jslint node: true */ +'use strict'; + +var binary = require('binary'); +var iconv = require('iconv-lite'); + +exports.readSAUCE = readSAUCE; + + +const SAUCE_SIZE = 128; +const SAUCE_ID = new Buffer([0x53, 0x41, 0x55, 0x43, 0x45]); // 'SAUCE' +const COMNT_ID = new Buffer([0x43, 0x4f, 0x4d, 0x4e, 0x54]); // 'COMNT' + +// :TODO: SAUCE should be a class +// - with getFontName() +// - ...other methods + +// +// See +// http://www.acid.org/info/sauce/sauce.htm +// +function readSAUCE(data, cb) { + if(data.length < SAUCE_SIZE) { + cb(new Error('No SAUCE record present')); + return; + } + + var offset = data.length - SAUCE_SIZE; + var sauceRec = data.slice(offset); + + binary.parse(sauceRec) + .buffer('id', 5) + .buffer('version', 2) + .buffer('title', 35) + .buffer('author', 20) + .buffer('group', 20) + .buffer('date', 8) + .word32lu('fileSize') + .word8('dataType') + .word8('fileType') + .word16lu('tinfo1') + .word16lu('tinfo2') + .word16lu('tinfo3') + .word16lu('tinfo4') + .word8('numComments') + .word8('flags') + .buffer('tinfos', 22) // SAUCE 00.5 + .tap(function onVars(vars) { + + if(!SAUCE_ID.equals(vars.id)) { + cb(new Error('No SAUCE record present')); + return; + } + + var ver = iconv.decode(vars.version, 'cp437'); + + if('00' !== ver) { + cb(new Error('Unsupported SAUCE version: ' + ver)); + return; + } + + var sauce = { + id : iconv.decode(vars.id, 'cp437'), + version : iconv.decode(vars.version, 'cp437').trim(), + title : iconv.decode(vars.title, 'cp437').trim(), + author : iconv.decode(vars.author, 'cp437').trim(), + group : iconv.decode(vars.group, 'cp437').trim(), + date : iconv.decode(vars.date, 'cp437').trim(), + fileSize : vars.fileSize, + dataType : vars.dataType, + fileType : vars.fileType, + tinfo1 : vars.tinfo1, + tinfo2 : vars.tinfo2, + tinfo3 : vars.tinfo3, + tinfo4 : vars.tinfo4, + numComments : vars.numComments, + flags : vars.flags, + tinfos : vars.tinfos, + }; + + var dt = SAUCE_DATA_TYPES[sauce.dataType]; + if(dt && dt.parser) { + sauce[dt.name] = dt.parser(sauce); + } + + cb(null, sauce); + }); +} + +// :TODO: These need completed: +var SAUCE_DATA_TYPES = {}; +SAUCE_DATA_TYPES[0] = { name : 'None' }; +SAUCE_DATA_TYPES[1] = { name : 'Character', parser : parseCharacterSAUCE }; +SAUCE_DATA_TYPES[2] = 'Bitmap'; +SAUCE_DATA_TYPES[3] = 'Vector'; +SAUCE_DATA_TYPES[4] = 'Audio'; +SAUCE_DATA_TYPES[5] = 'BinaryText'; +SAUCE_DATA_TYPES[6] = 'XBin'; +SAUCE_DATA_TYPES[7] = 'Archive'; +SAUCE_DATA_TYPES[8] = 'Executable'; + +var SAUCE_CHARACTER_FILE_TYPES = {}; +SAUCE_CHARACTER_FILE_TYPES[0] = 'ASCII'; +SAUCE_CHARACTER_FILE_TYPES[1] = 'ANSi'; +SAUCE_CHARACTER_FILE_TYPES[2] = 'ANSiMation'; +SAUCE_CHARACTER_FILE_TYPES[3] = 'RIP script'; +SAUCE_CHARACTER_FILE_TYPES[4] = 'PCBoard'; +SAUCE_CHARACTER_FILE_TYPES[5] = 'Avatar'; +SAUCE_CHARACTER_FILE_TYPES[6] = 'HTML'; +SAUCE_CHARACTER_FILE_TYPES[7] = 'Source'; +SAUCE_CHARACTER_FILE_TYPES[8] = 'TundraDraw'; + +// +// Map of SAUCE font -> encoding hint +// +// Note that this is the same mapping that x84 uses. Be compatible! +// +var SAUCE_FONT_TO_ENCODING_HINT = { + 'Amiga MicroKnight' : 'amiga', + 'Amiga MicroKnight+' : 'amiga', + 'Amiga mOsOul' : 'amiga', + 'Amiga P0T-NOoDLE' : 'amiga', + 'Amiga Topaz 1' : 'amiga', + 'Amiga Topaz 1+' : 'amiga', + 'Amiga Topaz 2' : 'amiga', + 'Amiga Topaz 2+' : 'amiga', + 'Atari ATASCII' : 'atari', + 'IBM EGA43' : 'cp437', + 'IBM EGA' : 'cp437', + 'IBM VGA25G' : 'cp437', + 'IBM VGA50' : 'cp437', + 'IBM VGA' : 'cp437', +}; + +['437', '720', '737', '775', '819', '850', '852', '855', '857', '858', +'860', '861', '862', '863', '864', '865', '866', '869', '872'].forEach(function onPage(page) { + var codec = 'cp' + page; + SAUCE_FONT_TO_ENCODING_HINT['IBM EGA43 ' + page] = codec; + SAUCE_FONT_TO_ENCODING_HINT['IBM EGA ' + page] = codec; + SAUCE_FONT_TO_ENCODING_HINT['IBM VGA25g ' + page] = codec; + SAUCE_FONT_TO_ENCODING_HINT['IBM VGA50 ' + page] = codec; + SAUCE_FONT_TO_ENCODING_HINT['IBM VGA ' + page] = codec; +}); + +function parseCharacterSAUCE(sauce) { + var result = {}; + + result.fileType = SAUCE_CHARACTER_FILE_TYPES[sauce.fileType] || 'Unknown'; + + if(sauce.fileType === 0 || sauce.fileType === 1 || sauce.fileType === 2) { + // convience: create ansiFlags + sauce.ansiFlags = sauce.flags; + + var i = 0; + while(i < sauce.tinfos.length && sauce.tinfos[i] !== 0x00) { + ++i; + } + var fontName = iconv.decode(sauce.tinfos.slice(0, i), 'cp437'); + if(fontName.length > 0) { + result.fontName = fontName; + } + } + + return result; +} \ No newline at end of file diff --git a/core/standard_menu.js b/core/standard_menu.js index d38a84f7..9848acb5 100644 --- a/core/standard_menu.js +++ b/core/standard_menu.js @@ -18,8 +18,8 @@ function StandardMenuModule(menuConfig) { require('util').inherits(StandardMenuModule, MenuModule); -StandardMenuModule.prototype.enter = function(client) { - StandardMenuModule.super_.prototype.enter.call(this, client); +StandardMenuModule.prototype.enter = function() { + StandardMenuModule.super_.prototype.enter.call(this); }; StandardMenuModule.prototype.beforeArt = function() { diff --git a/core/theme.js b/core/theme.js index 29498434..f23c6481 100644 --- a/core/theme.js +++ b/core/theme.js @@ -203,13 +203,13 @@ function getMergedTheme(menuConfig, promptConfig, theme) { } } - [ 'menus', 'prompts' ].forEach(function areaEntry(areaName) { - _.keys(mergedTheme[areaName]).forEach(function menuEntry(menuName) { + [ 'menus', 'prompts' ].forEach(function areaEntry(sectionName) { + _.keys(mergedTheme[sectionName]).forEach(function menuEntry(menuName) { var createdFormSection = false; - var mergedThemeMenu = mergedTheme[areaName][menuName]; + var mergedThemeMenu = mergedTheme[sectionName][menuName]; - if(_.has(theme, [ 'customization', areaName, menuName ])) { - var menuTheme = theme.customization[areaName][menuName]; + if(_.has(theme, [ 'customization', sectionName, menuName ])) { + var menuTheme = theme.customization[sectionName][menuName]; // config block is direct assign/overwrite // :TODO: should probably be _.merge() @@ -217,7 +217,7 @@ function getMergedTheme(menuConfig, promptConfig, theme) { mergedThemeMenu.config = _.assign(mergedThemeMenu.config || {}, menuTheme.config); } - if('menus' === areaName) { + if('menus' === sectionName) { if(_.isObject(mergedThemeMenu.form)) { getFormKeys(mergedThemeMenu.form).forEach(function formKeyEntry(formKey) { applyToForm(mergedThemeMenu.form[formKey], menuTheme, formKey); @@ -233,7 +233,7 @@ function getMergedTheme(menuConfig, promptConfig, theme) { createdFormSection = true; } } - } else if('prompts' === areaName) { + } else if('prompts' === sectionName) { // no 'form' or form keys for prompts -- direct to mci applyToForm(mergedThemeMenu, menuTheme); } @@ -247,7 +247,7 @@ function getMergedTheme(menuConfig, promptConfig, theme) { // * There is/was no explicit 'form' section // * There is no 'prompt' specified // - if('menus' === areaName && !_.isString(mergedThemeMenu.prompt) && + if('menus' === sectionName && !_.isString(mergedThemeMenu.prompt) && (createdFormSection || !_.isObject(mergedThemeMenu.form))) { mergedThemeMenu.runtime = _.merge(mergedThemeMenu.runtime || {}, { autoNext : true } ); diff --git a/core/user_login.js b/core/user_login.js index 72ca22e2..0c22490c 100644 --- a/core/user_login.js +++ b/core/user_login.js @@ -56,7 +56,7 @@ function userLogin(client, username, password, cb) { // update client logger with addition of username client.log = logger.log.child( { clientId : client.log.fields.clientId, username : user.username }); - client.log.info('Successful login'); + client.log.info('Successful login'); async.parallel( [ diff --git a/core/vertical_menu_view.js b/core/vertical_menu_view.js index 6e290969..c220b557 100644 --- a/core/vertical_menu_view.js +++ b/core/vertical_menu_view.js @@ -27,7 +27,7 @@ function VerticalMenuView(options) { this.dimens.height = Math.min(self.dimens.height, self.client.term.termHeight - self.position.row); } - if(this.autoScale.width) { + if(self.autoScale.width) { var l = 0; self.items.forEach(function item(i) { if(i.text.length > l) { @@ -148,6 +148,17 @@ VerticalMenuView.prototype.setFocus = function(focused) { VerticalMenuView.prototype.setFocusItemIndex = function(index) { VerticalMenuView.super_.prototype.setFocusItemIndex.call(this, index); // sets this.focusedItemIndex + + //this.updateViewVisibleItems(); + + // :TODO: |viewWindow| must be updated to reflect position change -- + // if > visibile then += by diff, if < visible + + if(this.focusedItemIndex > this.viewWindow.bottom) { + } else if (this.focusedItemIndex < this.viewWindow.top) { + // this.viewWindow.top--; +// this.viewWindow.bottom--; + } this.redraw(); }; diff --git a/core/view_controller.js b/core/view_controller.js index 7fbed5cd..8e4b36b9 100644 --- a/core/view_controller.js +++ b/core/view_controller.js @@ -488,7 +488,6 @@ ViewController.prototype.loadFromPromptConfig = function(options, cb) { assert(_.isObject(options.mciMap)); var self = this; - var promptName = _.isString(options.promptName) ? options.promptName : self.client.currentMenuModule.menuConfig.prompt; var promptConfig = _.isObject(options.config) ? options.config : self.client.currentMenuModule.menuConfig.promptConfig; var initialFocusId = 1; // default to first diff --git a/misc/acs_parser.pegjs b/misc/acs_parser.pegjs index dc9e339e..14a03042 100644 --- a/misc/acs_parser.pegjs +++ b/misc/acs_parser.pegjs @@ -16,15 +16,13 @@ return !isNaN(value) && user.getAge() >= value; }, AS : function accountStatus() { - if(_.isNumber(value)) { + if(!_.isArray(value)) { value = [ value ]; } - assert(_.isArray(value)); - - return _.findIndex(value, function cmp(accStatus) { - return parseInt(accStatus, 10) === parseInt(user.properties.account_status, 10); - }) > -1; + const userAccountStatus = parseInt(user.properties.account_status, 10); + value = value.map(n => parseInt(n, 10)); // ensure we have integers + return value.indexOf(userAccountStatus) > -1; }, EC : function isEncoding() { switch(value) { @@ -53,7 +51,7 @@ // :TODO: implement me!! return false; }, - SC : function isSecerConnection() { + SC : function isSecureConnection() { return client.session.isSecure; }, ML : function minutesLeft() { @@ -81,28 +79,20 @@ return !isNaN(value) && client.term.termWidth >= value; }, ID : function isUserId(value) { - if(_.isNumber(value)) { + if(!_.isArray(value)) { value = [ value ]; } - assert(_.isArray(value)); - - return _.findIndex(value, function cmp(uid) { - return user.userId === parseInt(uid, 10); - }) > -1; + value = value.map(n => parseInt(n, 10)); // ensure we have integers + return value.indexOf(user.userId) > -1; }, WD : function isOneOfDayOfWeek() { - if(_.isNumber(value)) { + if(!_.isArray(value)) { value = [ value ]; } - assert(_.isArray(value)); - - var nowDayOfWeek = new Date().getDay(); - - return _.findIndex(value, function cmp(dow) { - return nowDayOfWeek === parseInt(dow, 10); - }) > -1; + value = value.map(n => parseInt(n, 10)); // ensure we have integers + return value.indexOf(new Date().getDay()) > -1; }, MM : function isMinutesPastMidnight() { // :TODO: return true if value is >= minutes past midnight sys time diff --git a/mods/abracadabra.js b/mods/abracadabra.js index 5e5ff34d..c5aaafa6 100644 --- a/mods/abracadabra.js +++ b/mods/abracadabra.js @@ -177,12 +177,6 @@ function AbracadabraModule(options) { require('util').inherits(AbracadabraModule, MenuModule); -/* -AbracadabraModule.prototype.enter = function(client) { - AbracadabraModule.super_.prototype.enter.call(this, client); -}; -*/ - AbracadabraModule.prototype.leave = function() { AbracadabraModule.super_.prototype.leave.call(this); diff --git a/mods/menu.hjson b/mods/menu.hjson index de4f40da..4008a121 100644 --- a/mods/menu.hjson +++ b/mods/menu.hjson @@ -393,7 +393,7 @@ }, editorMode: edit editorType: email - messageAreaName: private_mail + messageAreaTag: private_mail toUserId: 1 /* always to +op */ } form: { @@ -806,7 +806,7 @@ }, editorMode: edit editorType: email - messageAreaName: private_mail + messageAreaTag: private_mail toUserId: 1 /* always to +op */ } form: { @@ -1019,6 +1019,10 @@ value: { command: "P" } action: @menu:messageAreaNewPost } + { + value: { command: "J" } + action: @menu:messageAreaChangeCurrentConference + } { value: { command: "C" } action: @menu:messageAreaChangeCurrentArea @@ -1041,7 +1045,39 @@ } ] } + + messageAreaChangeCurrentConference: { + art: CCHANGE + module: msg_conf_list + form: { + 0: { + mci: { + VM1: { + focus: true + submit: true + argName: conf + } + } + submit: { + *: [ + { + value: { conf: null } + action: @method:changeConference + } + ] + } + actionKeys: [ + { + keys: [ "escape", "q", "shift + q" ] + action: @systemMethod:prevMenu + } + ] + } + } + } + messageAreaChangeCurrentArea: { + // :TODO: rename this art to ACHANGE art: CHANGE module: msg_area_list form: { @@ -1070,6 +1106,7 @@ } } } + messageAreaMessageList: { module: msg_list art: MSGLIST diff --git a/mods/msg_area_list.js b/mods/msg_area_list.js index 4935e271..a984e9ff 100644 --- a/mods/msg_area_list.js +++ b/mods/msg_area_list.js @@ -5,7 +5,6 @@ var MenuModule = require('../core/menu_module.js').MenuModule; var ViewController = require('../core/view_controller.js').ViewController; var messageArea = require('../core/message_area.js'); var strUtil = require('../core/string_util.js'); -//var msgDb = require('./database.js').dbs.message; var async = require('async'); var assert = require('assert'); @@ -43,30 +42,33 @@ function MessageAreaListModule(options) { var self = this; - this.messageAreas = messageArea.getAvailableMessageAreas(); + this.messageAreas = messageArea.getSortedAvailMessageAreasByConfTag( + self.client.user.properties.message_conf_tag, + { client : self.client } + ); this.menuMethods = { changeArea : function(formData, extraArgs) { if(1 === formData.submitId) { - var areaName = self.messageAreas[formData.value.area].name; + const areaTag = self.messageAreas[formData.value.area].areaTag; - messageArea.changeMessageArea(self.client, areaName, function areaChanged(err) { - if(err) { - self.client.term.pipeWrite('\n|00Cannot change area: ' + err.message + '\n'); + messageArea.changeMessageArea(self.client, areaTag, function areaChanged(err) { + if(err) { + self.client.term.pipeWrite('\n|00Cannot change area: ' + err.message + '\n'); - setTimeout(function timeout() { - self.prevMenu(); - }, 1000); - } else { - self.prevMenu(); - } - }); + setTimeout(function timeout() { + self.prevMenu(); + }, 1000); + } else { + self.prevMenu(); + } + }); } } }; this.setViewText = function(id, text) { - var v = self.viewControllers.areaList.getView(id); + const v = self.viewControllers.areaList.getView(id); if(v) { v.setText(text); } @@ -78,7 +80,7 @@ require('util').inherits(MessageAreaListModule, MenuModule); MessageAreaListModule.prototype.mciReady = function(mciData, cb) { var self = this; - var vc = self.viewControllers.areaList = new ViewController( { client : self.client } ); + const vc = self.viewControllers.areaList = new ViewController( { client : self.client } ); async.series( [ @@ -99,26 +101,29 @@ MessageAreaListModule.prototype.mciReady = function(mciData, cb) { }); }, function populateAreaListView(callback) { - var listFormat = self.menuConfig.config.listFormat || '{index} ) - {desc}'; - var focusListFormat = self.menuConfig.config.focusListFormat || listFormat; - - var areaListItems = []; - var focusListItems = []; - - // :TODO: use _.map() here - for(var i = 0; i < self.messageAreas.length; ++i) { - areaListItems.push(listFormat.format( - { index : i, name : self.messageAreas[i].name, desc : self.messageAreas[i].desc } ) - ); - focusListItems.push(focusListFormat.format( - { index : i, name : self.messageAreas[i].name, desc : self.messageAreas[i].desc } ) - ); - } - - var areaListView = vc.getView(1); - - areaListView.setItems(areaListItems); - areaListView.setFocusItems(focusListItems); + const listFormat = self.menuConfig.config.listFormat || '{index} ) - {name}'; + const focusListFormat = self.menuConfig.config.focusListFormat || listFormat; + + const areaListView = vc.getView(1); + let i = 1; + areaListView.setItems(_.map(self.messageAreas, v => { + return listFormat.format({ + index : i++, + areaTag : v.area.areaTag, + name : v.area.name, + desc : v.area.desc, + }); + })); + + i = 1; + areaListView.setFocusItems(_.map(self.messageAreas, v => { + return focusListFormat.format({ + index : i++, + areaTag : v.area.areaTag, + name : v.area.name, + desc : v.area.desc, + }) + })); areaListView.redraw(); diff --git a/mods/msg_area_post_fse.js b/mods/msg_area_post_fse.js index 4723da68..2b0c488d 100644 --- a/mods/msg_area_post_fse.js +++ b/mods/msg_area_post_fse.js @@ -56,11 +56,11 @@ function AreaPostFSEModule(options) { require('util').inherits(AreaPostFSEModule, FullScreenEditorModule); -AreaPostFSEModule.prototype.enter = function(client) { +AreaPostFSEModule.prototype.enter = function() { - if(_.isString(client.user.properties.message_area_name) && !_.isString(this.messageAreaName)) { - this.messageAreaName = client.user.properties.message_area_name; + if(_.isString(this.client.user.properties.message_area_tag) && !_.isString(this.messageAreaTag)) { + this.messageAreaTag = this.client.user.properties.message_area_tag; } - AreaPostFSEModule.super_.prototype.enter.call(this, client); + AreaPostFSEModule.super_.prototype.enter.call(this); }; diff --git a/mods/msg_area_view_fse.js b/mods/msg_area_view_fse.js index 7d1d1e44..09d823ed 100644 --- a/mods/msg_area_view_fse.js +++ b/mods/msg_area_view_fse.js @@ -72,7 +72,7 @@ function AreaViewFSEModule(options) { if(_.isString(extraArgs.menu)) { var modOpts = { extraArgs : { - messageAreaName : self.messageAreaName, + messageAreaTag : self.messageAreaTag, replyToMessage : self.message, } }; diff --git a/mods/msg_conf_list.js b/mods/msg_conf_list.js new file mode 100644 index 00000000..308785d5 --- /dev/null +++ b/mods/msg_conf_list.js @@ -0,0 +1,122 @@ +/* jslint node: true */ +'use strict'; + +var MenuModule = require('../core/menu_module.js').MenuModule; +var ViewController = require('../core/view_controller.js').ViewController; +var messageArea = require('../core/message_area.js'); + +var async = require('async'); +var assert = require('assert'); +var _ = require('lodash'); + +exports.getModule = MessageConfListModule; + +exports.moduleInfo = { + name : 'Message Conference List', + desc : 'Module for listing / choosing message conferences', + author : 'NuSkooler', +}; + +var MciCodesIds = { + ConfList : 1, + CurrentConf : 2, + + // :TODO: + // # areas in con + // +}; + +function MessageConfListModule(options) { + MenuModule.call(this, options); + + var self = this; + + this.messageConfs = messageArea.getSortedAvailMessageConferences(self.client); + + this.menuMethods = { + changeConference : function(formData, extraArgs) { + if(1 === formData.submitId) { + const confTag = self.messageConfs[formData.value.conf].confTag; + + messageArea.changeMessageConference(self.client, confTag, err => { + if(err) { + self.client.term.pipeWrite(`\n|00Cannot change conference: ${err.message}\n`); + + setTimeout(function timeout() { + self.prevMenu(); + }, 1000); + } else { + self.prevMenu(); + } + }); + } + } + }; + + this.setViewText = function(id, text) { + const v = self.viewControllers.areaList.getView(id); + if(v) { + v.setText(text); + } + }; +} + +require('util').inherits(MessageConfListModule, MenuModule); + +MessageConfListModule.prototype.mciReady = function(mciData, cb) { + var self = this; + const vc = self.viewControllers.areaList = new ViewController( { client : self.client } ); + + async.series( + [ + function callParentMciReady(callback) { + MessageConfListModule.super_.prototype.mciReady.call(this, mciData, callback); + }, + function loadFromConfig(callback) { + let loadOpts = { + callingMenu : self, + mciMap : mciData.menu, + formId : 0, + }; + + vc.loadFromMenuConfig(loadOpts, callback); + }, + function populateConfListView(callback) { + const listFormat = self.menuConfig.config.listFormat || '{index} ) - {name}'; + const focusListFormat = self.menuConfig.config.focusListFormat || listFormat; + + const confListView = vc.getView(1); + let i = 1; + confListView.setItems(_.map(self.messageConfs, v => { + return listFormat.format({ + index : i++, + confTag : v.conf.confTag, + name : v.conf.name, + desc : v.conf.desc, + }); + })); + + i = 1; + confListView.setFocusItems(_.map(self.messageConfs, v => { + return focusListFormat.format({ + index : i++, + confTag : v.conf.confTag, + name : v.conf.name, + desc : v.conf.desc, + }) + })); + + confListView.redraw(); + + callback(null); + }, + function populateTextViews(callback) { + // :TODO: populate other avail MCI, e.g. current conf name + callback(null); + } + ], + function complete(err) { + cb(err); + } + ); +}; \ No newline at end of file diff --git a/mods/msg_list.js b/mods/msg_list.js index c5fbe768..fbffa162 100644 --- a/mods/msg_list.js +++ b/mods/msg_list.js @@ -52,15 +52,15 @@ function MessageListModule(options) { var self = this; var config = this.menuConfig.config; - this.messageAreaName = config.messageAreaName; + this.messageAreaTag = config.messageAreaTag; if(options.extraArgs) { // - // |extraArgs| can override |messageAreaName| provided by config + // |extraArgs| can override |messageAreaTag| provided by config // as well as supply a pre-defined message list // - if(options.extraArgs.messageAreaName) { - this.messageAreaName = options.extraArgs.messageAreaName; + if(options.extraArgs.messageAreaTag) { + this.messageAreaTag = options.extraArgs.messageAreaTag; } if(options.extraArgs.messageList) { @@ -73,7 +73,7 @@ function MessageListModule(options) { if(1 === formData.submitId) { var modOpts = { extraArgs : { - messageAreaName : self.messageAreaName, + messageAreaTag : self.messageAreaTag, messageList : self.messageList, messageIndex : formData.value.message, } @@ -94,15 +94,15 @@ function MessageListModule(options) { require('util').inherits(MessageListModule, MenuModule); -MessageListModule.prototype.enter = function(client) { - MessageListModule.super_.prototype.enter.call(this, client); +MessageListModule.prototype.enter = function() { + MessageListModule.super_.prototype.enter.call(this); // - // Config can specify |messageAreaName| else it comes from + // Config can specify |messageAreaTag| else it comes from // the user's current area // - if(!this.messageAreaName) { - this.messageAreaName = client.user.properties.message_area_name; + if(!this.messageAreaTag) { + this.messageAreaTag = this.client.user.properties.message_area_tag; } }; @@ -110,6 +110,8 @@ MessageListModule.prototype.mciReady = function(mciData, cb) { var self = this; var vc = self.viewControllers.allViews = new ViewController( { client : self.client } ); + var firstNewEntryIndex; + async.series( [ function callParentMciReady(callback) { @@ -130,7 +132,7 @@ MessageListModule.prototype.mciReady = function(mciData, cb) { if(_.isArray(self.messageList)) { callback(0 === self.messageList.length ? new Error('No messages in area') : null); } else { - messageArea.getMessageListForArea( { client : self.client }, self.messageAreaName, function msgs(err, msgList) { + messageArea.getMessageListForArea( { client : self.client }, self.messageAreaTag, function msgs(err, msgList) { if(msgList && 0 === msgList.length) { callback(new Error('No messages in area')); } else { @@ -141,7 +143,7 @@ MessageListModule.prototype.mciReady = function(mciData, cb) { } }, function getLastReadMesageId(callback) { - messageArea.getMessageAreaLastReadId(self.client.user.userId, self.messageAreaName, function lastRead(err, lastReadId) { + messageArea.getMessageAreaLastReadId(self.client.user.userId, self.messageAreaTag, function lastRead(err, lastReadId) { self.lastReadId = lastReadId || 0; callback(null); // ignore any errors, e.g. missing value }); @@ -158,6 +160,13 @@ MessageListModule.prototype.mciReady = function(mciData, cb) { var msgNum = 1; function getMsgFmtObj(mle) { + + if(_.isUndefined(firstNewEntryIndex) && + mle.messageId > self.lastReadId) + { + firstNewEntryIndex = msgNum - 1; + } + return { msgNum : msgNum++, subj : mle.subject, @@ -180,14 +189,18 @@ MessageListModule.prototype.mciReady = function(mciData, cb) { msgListView.on('index update', function indexUpdated(idx) { self.setViewText(MciCodesIds.MsgSelNum, (idx + 1).toString()); }); - + msgListView.redraw(); + + if(firstNewEntryIndex > 0) { + msgListView.setFocusItemIndex(firstNewEntryIndex); + } callback(null); }, function populateOtherMciViews(callback) { - self.setViewText(MciCodesIds.MsgAreaDesc, messageArea.getMessageAreaByName(self.messageAreaName).desc); + self.setViewText(MciCodesIds.MsgAreaDesc, messageArea.getMessageAreaByTag(self.messageAreaTag).name); self.setViewText(MciCodesIds.MsgSelNum, (vc.getView(MciCodesIds.MsgList).getData() + 1).toString()); self.setViewText(MciCodesIds.MsgTotal, self.messageList.length.toString()); diff --git a/mods/nua.js b/mods/nua.js index c2a955f2..7d83e518 100644 --- a/mods/nua.js +++ b/mods/nua.js @@ -5,7 +5,7 @@ var user = require('../core/user.js'); var theme = require('../core/theme.js'); var login = require('../core/system_menu_method.js').login; var Config = require('../core/config.js').config; -var getDefaultMessageArea = require('../core/message_area.js').getDefaultMessageArea; +var messageArea = require('../core/message_area.js'); var async = require('async'); @@ -65,6 +65,16 @@ function NewUserAppModule(options) { newUser.username = formData.value.username; + // + // We have to disable ACS checks for initial default areas as the user is not yet ready + // + var confTag = messageArea.getDefaultMessageConferenceTag(self.client, true); // true=disableAcsCheck + var areaTag = messageArea.getDefaultMessageAreaTagByConfTag(self.client, confTag, true); // true=disableAcsCheck + + // can't store undefined! + confTag = confTag || ''; + areaTag = areaTag || ''; + newUser.properties = { real_name : formData.value.realName, birthdate : new Date(Date.parse(formData.value.birthdate)).toISOString(), @@ -74,14 +84,12 @@ function NewUserAppModule(options) { email_address : formData.value.email, web_address : formData.value.web, account_created : new Date().toISOString(), - - message_area_name : getDefaultMessageArea().name, + + message_conf_tag : confTag, + message_area_tag : areaTag, term_height : self.client.term.termHeight, - term_width : self.client.term.termWidth, - - // :TODO: This is set in User.create() -- proabbly don't need it here: - //account_status : Config.users.requireActivation ? user.User.AccountStatus.inactive : user.User.AccountStatus.active, + term_width : self.client.term.termWidth, // :TODO: Other defaults // :TODO: should probably have a place to create defaults/etc. @@ -92,8 +100,8 @@ function NewUserAppModule(options) { } else { newUser.properties.theme_id = Config.defaults.theme; } - - // :TODO: .create() should also validate email uniqueness! + + // :TODO: User.create() should validate email uniqueness! newUser.create( { password : formData.value.password }, function created(err) { if(err) { self.client.log.info( { error : err, username : formData.value.username }, 'New user creation failed'); diff --git a/mods/themes/luciano_blocktronics/theme.hjson b/mods/themes/luciano_blocktronics/theme.hjson index 60d56059..44bd8131 100644 --- a/mods/themes/luciano_blocktronics/theme.hjson +++ b/mods/themes/luciano_blocktronics/theme.hjson @@ -172,8 +172,8 @@ messageAreaChangeCurrentArea: { config: { - listFormat: "|00|15{index} |07- |03{desc}" - focusListFormat: "|00|19|15{index} - {desc}" + listFormat: "|00|15{index} |07- |03{name}" + focusListFormat: "|00|19|15{index} - {name}" } mci: { VM1: { @@ -310,6 +310,19 @@ } } } + + newScanMessageList: { + config: { + listFormat: "|00|15 {msgNum:<5.5}|03{subj:<29.29} |15{from:<20.20} {ts}" + focusListFormat: "|00|19> |15{msgNum:<5.5}{subj:<29.29} {from:<20.20} {ts}" + dateTimeFormat: ddd MMM Do + } + mci: { + VM1: { + height: 14 + } + } + } } } } \ No newline at end of file diff --git a/mods/user_list.js b/mods/user_list.js index 59826753..5303e420 100644 --- a/mods/user_list.js +++ b/mods/user_list.js @@ -2,7 +2,7 @@ 'use strict'; var MenuModule = require('../core/menu_module.js').MenuModule; -var userDb = require('../core/database.js').dbs.user; +//var userDb = require('../core/database.js').dbs.user; var getUserList = require('../core/user.js').getUserList; var ViewController = require('../core/view_controller.js').ViewController; diff --git a/mods/whos_online.js b/mods/whos_online.js index 6b0ef4c1..a607ae02 100644 --- a/mods/whos_online.js +++ b/mods/whos_online.js @@ -84,7 +84,6 @@ WhosOnlineModule.prototype.mciReady = function(mciData, cb) { return listFormat.format(oe); })); - // :TODO: This is a hack until pipe codes are better implemented onlineListView.focusItems = onlineListView.items; onlineListView.redraw(); diff --git a/oputil.js b/oputil.js index 5eef5339..5087b9b1 100755 --- a/oputil.js +++ b/oputil.js @@ -13,7 +13,7 @@ var assert = require('assert'); var argv = require('minimist')(process.argv.slice(2)); -var ExitCodes = { +const ExitCodes = { SUCCESS : 0, ERROR : -1, BAD_COMMAND : -2, @@ -28,9 +28,13 @@ function printUsage(command) { usage = 'usage: oputil.js [--version] [--help]\n' + ' []' + - '\n' + + '\n\n' + 'global args:\n' + - ' --config PATH : specify config path'; + ' --config PATH : specify config path' + + '\n\n' + + 'commands:\n' + + ' user : User utilities' + + '\n'; break; case 'user' : @@ -47,7 +51,7 @@ function printUsage(command) { } function initConfig(cb) { - var configPath = argv.config ? argv.config : config.getDefaultPath(); + const configPath = argv.config ? argv.config : config.getDefaultPath(); config.init(configPath, cb); } @@ -88,7 +92,7 @@ function handleUserCommand() { assert(_.isNumber(userId)); assert(userId > 0); - var u = new user.User(); + let u = new user.User(); u.userId = userId; u.setNewAuthCredentials(argv.password, function credsSet(err) { From dec78e942d9df7b88da4a21e65e44c3e93f358dd Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 9 Feb 2016 22:30:59 -0700 Subject: [PATCH 02/27] * Reworked FTN packet I/O (WIP) * Detect FTN packet 2, 2.2, and 2+ * Various FTN utils (MSGID, Origin, PID, generation etc) * More work on message network readyness --- core/config.js | 5 +- core/fse.js | 4 +- core/ftn_mail_packet.js | 704 +++----------------- core/ftn_util.js | 239 +++++-- core/message.js | 2 +- core/msg_network_module.js | 25 - core/msg_networks/ftn_msg_network_module.js | 27 - core/msg_scan_toss_module.js | 25 + core/sauce.js | 9 +- core/scanner_tossers/ftn_bso.js | 42 ++ core/theme.js | 3 +- 11 files changed, 377 insertions(+), 708 deletions(-) delete mode 100644 core/msg_network_module.js delete mode 100644 core/msg_networks/ftn_msg_network_module.js create mode 100644 core/msg_scan_toss_module.js create mode 100644 core/scanner_tossers/ftn_bso.js diff --git a/core/config.js b/core/config.js index dc8f3a8e..61de4244 100644 --- a/core/config.js +++ b/core/config.js @@ -172,7 +172,10 @@ function getDefaultConfig() { paths : { mods : paths.join(__dirname, './../mods/'), servers : paths.join(__dirname, './servers/'), - msgNetworks : paths.join(__dirname, './msg_networks/'), + + scannerTossers : paths.join(__dirname, './scanner_tossers/'), + mailers : paths.join(__dirname, './mailers/') , + art : paths.join(__dirname, './../mods/art/'), themes : paths.join(__dirname, './../mods/themes/'), logs : paths.join(__dirname, './../logs/'), // :TODO: set up based on system, e.g. /var/logs/enigmabbs or such diff --git a/core/fse.js b/core/fse.js index 69c5e418..317485e9 100644 --- a/core/fse.js +++ b/core/fse.js @@ -307,9 +307,9 @@ function FullScreenEditorModule(options) { // :TODO: We'd like to delete up to N rows, but this does not work // in NetRunner: - //self.client.term.rawWrite(ansi.reset() + ansi.deleteLine(3)); + self.client.term.rawWrite(ansi.reset() + ansi.deleteLine(3)); - self.client.term.rawWrite(ansi.reset() + ansi.eraseLine(2)) + //self.client.term.rawWrite(ansi.reset() + ansi.eraseLine(2)) } callback(null); }, diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index a9603f2e..8b69228d 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -17,39 +17,37 @@ var buffers = require('buffers'); var moment = require('moment'); /* - :TODO: should probably be broken up - FTNPacket - FTNPacketImport: packet -> message(s) - FTNPacketExport: message(s) -> packet -*/ + :TODO: things + * Read/detect packet types: 2, 2.2, and 2+ + * Write packet types: 2, 2.2, and 2+ + * Test SAUCE ignore/extraction + * FSP-1010 for netmail (see SBBS) -/* -Reader: file to ftn data -Writer: ftn data to packet -Data to toMessage -Data.fromMessage - -FTNMessage.toMessage() => Message -FTNMessage.fromMessage() => Create from Message - -* read: header -> simple {} obj, msg -> Message object -* read: read(..., iterator): iterator('header', ...), iterator('message', msg) -* write: provide information to go into header - -* Logic of "Is this for us"/etc. elsewhere */ const FTN_PACKET_HEADER_SIZE = 58; // fixed header size const FTN_PACKET_HEADER_TYPE = 2; const FTN_PACKET_MESSAGE_TYPE = 2; +const FTN_PACKET_BAUD_TYPE_2_2 = 2; -// EOF + SAUCE.id + SAUCE.version ('00') -const FTN_MESSAGE_SAUCE_HEADER = - new Buffer( [ 0x1a, 'S', 'A', 'U', 'C', 'E', '0', '0' ] ); +// SAUCE magic header + version ("00") +const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; +// +// Read/Write FTN packets with support for the following formats: +// +// * Type 1 FTS-0001 @ http://ftsc.org/docs/fts-0001.016 (Obsolete) +// * Type 2.2 FSC-0045 @ http://ftsc.org/docs/fsc-0045.001 +// * Type 2+ FSC-0039 and FSC-0048 @ http://ftsc.org/docs/fsc-0039.004 +// and http://ftsc.org/docs/fsc-0048.002 +// +// Additional resources: +// * Writeup on differences between type 2, 2.2, and 2+: +// http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt +// function FTNPacket() { var self = this; @@ -57,16 +55,14 @@ function FTNPacket() { this.parsePacketHeader = function(packetBuffer, cb) { assert(Buffer.isBuffer(packetBuffer)); - // - // See the following specs: - // http://ftsc.org/docs/fts-0001.016 - // http://ftsc.org/docs/fsc-0048.002 - // if(packetBuffer.length < FTN_PACKET_HEADER_SIZE) { cb(new Error('Buffer too small')); return; } + // + // Start out reading as if this is a FSC-0048 2+ packet + // binary.parse(packetBuffer) .word16lu('origNode') .word16lu('destNode') @@ -81,7 +77,7 @@ function FTNPacket() { .word16lu('origNet') .word16lu('destNet') .word8('prodCodeLo') - .word8('revisionMajor') // aka serialNo + .word8('prodRevLo') // aka serialNo .buffer('password', 8) // null padded C style string .word16lu('origZone') .word16lu('destZone') @@ -89,9 +85,9 @@ function FTNPacket() { .word16lu('auxNet') .word16lu('capWordA') .word8('prodCodeHi') - .word8('revisionMinor') + .word8('prodRevHi') .word16lu('capWordB') - .word16lu('originZone2') + .word16lu('origZone2') .word16lu('destZone2') .word16lu('originPoint') .word16lu('destPoint') @@ -104,6 +100,30 @@ function FTNPacket() { cb(new Error('Unsupported header type: ' + packetHeader.packetType)); return; } + + // + // What kind of packet do we really have here? + // + if(FTN_PACKET_BAUD_TYPE_2_2 === packetHeader.baud) { + packetHeader.packetVersion = '2.2'; + } else { + // + // See heuristics described in FSC-0048, "Receiving Type-2+ bundles" + // + const capWordASwapped = + ((packetHeader.capWordA & 0xff) << 8) | + ((packetHeader.capWordA >> 8) & 0xff); + + if(capWordASwapped === packetHeader.capWordB && + 0 != packetHeader.capWordB && + packetHeader.capWordB & 0x0001) + { + packetHeader.packetVersion = '2+'; + } else { + packetHeader.packetVersion = '2'; + packetHeader.point + } + } // // Date/time components into something more reasonable @@ -131,7 +151,7 @@ function FTNPacket() { buffer.writeUInt16LE(headerInfo.origNet, 20); buffer.writeUInt16LE(headerInfo.destNet, 22); buffer.writeUInt8(headerInfo.prodCodeLo, 24); - buffer.writeUInt8(headerInfo.revisionMajor, 25); + buffer.writeUInt8(headerInfo.prodRevHi, 25); const pass = ftn.stringToNullPaddedBuffer(headerInfo.password, 8); pass.copy(buffer, 26); @@ -143,7 +163,7 @@ function FTNPacket() { buffer.writeUInt16LE(headerInfo.auxNet, 38); buffer.writeUInt16LE(headerInfo.capWordA, 40); buffer.writeUInt8(headerInfo.prodCodeHi, 42); - buffer.writeUInt8(headerInfo.revisionMinor, 43); + buffer.writeUInt8(headerInfo.prodRevLo, 43); buffer.writeUInt16LE(headerInfo.capWordB, 44); buffer.writeUInt16LE(headerInfo.origZone2, 46); buffer.writeUInt16LE(headerInfo.destZone2, 48); @@ -207,13 +227,16 @@ function FTNPacket() { // :TODO: This is wrong: SAUCE may not have EOF marker for one, also if it's // present, we need to extract it but keep the rest of hte message intact as it likely // has SEEN-BY, PATH, and other kludge information *appended* - const sauceHeaderPosition = messageBodyBuffer.indexOf(FTN_MESSAGE_SAUCE_HEADER); + const sauceHeaderPosition = messageBodyBuffer.indexOf(FTN_MESSAGE_SAUCE_HEADER); if(sauceHeaderPosition > -1) { - sauce.readSAUCE(messageBodyBuffer.slice(sauceHeaderPosition), (err, theSauce) => { + sauce.readSAUCE(messageBodyBuffer.slice(sauceHeaderPosition, sauceHeaderPosition + sauce.SAUCE_SIZE), (err, theSauce) => { if(!err) { // we read some SAUCE - don't re-process that portion into the body - messageBodyBuffer = messageBodyBuffer.slice(0, sauceHeaderPosition); + messageBodyBuffer = messageBodyBuffer.slice(0, sauceHeaderPosition) + messageBodyBuffer.slice(sauceHeaderPosition + sauce.SAUCE_SIZE); +// messageBodyBuffer = messageBodyBuffer.slice(0, sauceHeaderPosition); messageBodyData.sauce = theSauce; + } else { + console.log(err) } callback(null); // failure to read SAUCE is OK }); @@ -349,6 +372,19 @@ function FTNPacket() { msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine; } + // + // Update message UUID, if possible, based on MSGID and AREA + // + if(_.isString(msg.meta.FtnKludge.MSGID) && + _.isString(msg.meta.FtnProperty.ftn_area) && + msg.meta.FtnKludge.MSGID.length > 0 && + msg.meta.FtnProperty.ftn_area.length > 0) + { + msg.uuid = ftn.createMessageUuid( + msg.meta.FtnKludge.MSGID, + msg.meta.FtnProperty.area); + } + iterator('message', msg); }) }); @@ -367,21 +403,6 @@ function FTNPacket() { basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags2, 11); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); - // - // From http://ftsc.org/docs/fts-0001.016: - // DateTime = (* a character string 20 characters long *) - // (* 01 Jan 86 02:34:56 *) - // DayOfMonth " " Month " " Year " " - // " " HH ":" MM ":" SS - // Null - // - // DayOfMonth = "01" | "02" | "03" | ... | "31" (* Fido 0 fills *) - // Month = "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | - // "Jul" | "Aug" | "Sep" | "Oct" | "Nov" | "Dec" - // Year = "01" | "02" | .. | "85" | "86" | ... | "99" | "00" - // HH = "00" | .. | "23" - // MM = "00" | .. | "59" - // SS = "00" | .. | "59" const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); dateTimeBuffer.copy(basicHeader, 14); @@ -423,12 +444,17 @@ function FTNPacket() { } } - // :TODO: is Area really any differnt (e.g. no space between AREA:the_area) + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // AREA:CONFERENCE + // Should be first line in a message + // if(message.meta.FtnProperty.ftn_area) { msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\n`; } Object.keys(message.meta.FtnKludge).forEach(k => { + // we want PATH to be last if('PATH' !== k) { appendMeta(k, message.meta.FtnKludge[k]); } @@ -436,9 +462,21 @@ function FTNPacket() { msgBody += message.message; + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // Origin line should be near the bottom of a message + // appendMeta('', message.meta.FtnProperty.ftn_tear_line); + + // + // Tear line should be near the bottom of a message + // appendMeta('', message.meta.FtnProperty.ftn_origin); + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // SEEN-BY and PATH should be the last lines of a message + // appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); appendMeta('PATH', message.meta.FtnKludge['PATH']); @@ -509,498 +547,6 @@ FTNPacket.prototype.write = function(path, headerInfo, messages, cb) { }; - -// -// References -// * http://ftsc.org/docs/fts-0001.016 -// * http://ftsc.org/docs/fsc-0048.002 -// -// Other implementations: -// * https://github.com/M-griffin/PyPacketMail/blob/master/PyPacketMail.py -// -function FTNMailPacket(options) { - - //MailPacket.call(this, options); - - var self = this; - self.KLUDGE_PREFIX = '\x01'; - - this.getPacketHeaderAddress = function() { - return { - zone : self.packetHeader.destZone, - net : self.packetHeader.destNet, - node : self.packetHeader.destNode, - point : self.packetHeader.destPoint, - }; - }; - - this.getNetworkNameForAddress = function(addr) { - var nodeAddr; - for(var network in self.nodeAddresses) { - nodeAddr = self.nodeAddresses[network]; - if(nodeAddr.zone === addr.zone && - nodeAddr.net === addr.net && - nodeAddr.node === addr.node && - nodeAddr.point === addr.point) - { - return network; - } - } - }; - - this.parseFtnPacketHeader = function(packetBuffer, cb) { - assert(Buffer.isBuffer(packetBuffer)); - - if(packetBuffer.length < 58) { - cb(new Error('Buffer too small')); - return; - } - - binary.parse(packetBuffer) - .word16lu('origNode') - .word16lu('destNode') - .word16lu('year') - .word16lu('month') - .word16lu('day') - .word16lu('hour') - .word16lu('minute') - .word16lu('second') - .word16lu('baud') - .word16lu('packetType') - .word16lu('origNet') - .word16lu('destNet') - .word8('prodCodeLo') - .word8('revisionMajor') // aka serialNo - .buffer('password', 8) // null terminated C style string - .word16lu('origZone') - .word16lu('destZone') - // Additions in FSC-0048.002 follow... - .word16lu('auxNet') - .word16lu('capWordA') - .word8('prodCodeHi') - .word8('revisionMinor') - .word16lu('capWordB') - .word16lu('originZone2') - .word16lu('destZone2') - .word16lu('originPoint') - .word16lu('destPoint') - .word32lu('prodData') - .tap(function tapped(packetHeader) { - packetHeader.password = ftn.stringFromFTN(packetHeader.password); - - // :TODO: Don't hard code magic # here - if(2 !== packetHeader.packetType) { - console.log(packetHeader.packetType) - cb(new Error('Packet is not Type-2')); - return; - } - - // :TODO: convert date information -> .created - - packetHeader.created = moment(packetHeader); - /* - packetHeader.year, packetHeader.month, packetHeader.day, packetHeader.hour, - packetHeader.minute, packetHeader.second);*/ - - // :TODO: validate & pass error if failure - cb(null, packetHeader); - }); - }; - - this.getPacketHeaderBuffer = function(packetHeader, options) { - options = options || {}; - - if(options.created) { - options.created = moment(options.created); // ensure we have a moment obj - } else { - options.created = moment(); - } - - let buffer = new Buffer(58); - - buffer.writeUInt16LE(packetHeader.origNode, 0); - buffer.writeUInt16LE(packetHeader.destNode, 2); - buffer.writeUInt16LE(options.created.year(), 4); - buffer.writeUInt16LE(options.created.month(), 6); - buffer.writeUInt16LE(options.created.date(), 8); - buffer.writeUInt16LE(options.created.hour(), 10); - buffer.writeUInt16LE(options.created.minute(), 12); - buffer.writeUInt16LE(options.created.second(), 14); - buffer.writeUInt16LE(0x0000, 16); - buffer.writeUInt16LE(0x0002, 18); - buffer.writeUInt16LE(packetHeader.origNet, 20); - buffer.writeUInt16LE(packetHeader.destNet, 22); - buffer.writeUInt8(packetHeader.prodCodeLo, 24); - buffer.writeUInt8(packetHeader.revisionMajor, 25); - - const pass = ftn.stringToNullPaddedBuffer(packetHeader.password, 8); - pass.copy(buffer, 26); - - buffer.writeUInt16LE(packetHeader.origZone, 34); - buffer.writeUInt16LE(packetHeader.destZone, 36); - - // FSC-0048.002 additions... - buffer.writeUInt16LE(packetHeader.auxNet, 38); - buffer.writeUInt16LE(packetHeader.capWordA, 40); - buffer.writeUInt8(packetHeader.prodCodeHi, 42); - buffer.writeUInt8(packetHeader.revisionMinor, 43); - buffer.writeUInt16LE(packetHeader.capWordB, 44); - buffer.writeUInt16LE(packetHeader.origZone2, 46); - buffer.writeUInt16LE(packetHeader.destZone2, 48); - buffer.writeUInt16LE(packetHeader.origPoint, 50); - buffer.writeUInt16LE(packetHeader.destPoint, 52); - buffer.writeUInt32LE(packetHeader.prodData, 54); - - return buffer; - }; - - self.setOrAppend = function(value, dst) { - if(dst) { - if(!_.isArray(dst)) { - dst = [ dst ]; - } - - dst.push(value); - } else { - dst = value; - } - } - - - self.getMessageMeta = function(msgBody, msgData) { - var meta = { - FtnKludge : msgBody.kludgeLines, - FtnProperty : {}, - }; - - if(msgBody.tearLine) { - meta.FtnProperty.ftn_tear_line = msgBody.tearLine; - } - if(msgBody.seenBy.length > 0) { - meta.FtnProperty.ftn_seen_by = msgBody.seenBy; - } - if(msgBody.area) { - meta.FtnProperty.ftn_area = msgBody.area; - } - if(msgBody.originLine) { - meta.FtnProperty.ftn_origin = msgBody.originLine; - } - - meta.FtnProperty.ftn_orig_node = msgData.origNode; - meta.FtnProperty.ftn_dest_node = msgData.destNode; - meta.FtnProperty.ftn_orig_network = msgData.origNet; - meta.FtnProperty.ftn_dest_network = msgData.destNet; - meta.FtnProperty.ftn_attr_flags1 = msgData.attrFlags1; - meta.FtnProperty.ftn_attr_flags2 = msgData.attrFlags2; - meta.FtnProperty.ftn_cost = msgData.cost; - - return meta; - }; - - this.parseFtnMessageBody = function(msgBodyBuffer, cb) { - // - // From FTS-0001.16: - // "Message text is unbounded and null terminated (note exception below). - // - // A 'hard' carriage return, 0DH, marks the end of a paragraph, and must - // be preserved. - // - // So called 'soft' carriage returns, 8DH, may mark a previous - // processor's automatic line wrap, and should be ignored. Beware that - // they may be followed by linefeeds, or may not. - // - // All linefeeds, 0AH, should be ignored. Systems which display message - // text should wrap long lines to suit their application." - // - // This is a bit tricky. Decoding the buffer to CP437 converts all 0x8d -> 0xec, so we'll - // have to replace those characters if the buffer is left as CP437. - // After decoding, we'll need to peek at the buffer for the various kludge lines - // for charsets & possibly re-decode. Uggh! - // - - // :TODO: Use the proper encoding here. There appear to be multiple specs and/or - // stuff people do with this... some specs kludge lines, which is kinda durpy since - // to get to that point, one must read the file (and decode) to find said kludge... - - - //var msgLines = msgBodyBuffer.toString().split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g); - - //var msgLines = iconv.decode(msgBodyBuffer, 'CP437').replace(/\xec/g, '').split(/\r\n|[\r\n]/g); - var msgLines = iconv.decode(msgBodyBuffer, 'CP437').replace(/[\xec\n]/g, '').split(/\r/g); - - var msgBody = { - message : [], - kludgeLines : {}, // -> [ value1, value2, ... ] - seenBy : [], - }; - - var preOrigin = true; - - function addKludgeLine(kl) { - const kludgeParts = kl.split(':'); - kludgeParts[0] = kludgeParts[0].toUpperCase(); - kludgeParts[1] = kludgeParts[1].trim(); - - self.setOrAppend(kludgeParts[1], msgBody.kludgeLines[kludgeParts[0]]); - } - - var sauceBuffers; - - msgLines.forEach(function nextLine(line) { - if(0 === line.length) { - msgBody.message.push(''); - return; - } - - if(preOrigin) { - if(_.startsWith(line, 'AREA:')) { - msgBody.area = line.substring(line.indexOf(':') + 1).trim(); - } else if(_.startsWith(line, '--- ')) { - // Tag lines are tracked allowing for specialized display/etc. - msgBody.tearLine = line; - } else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) { // To spec is " * Origin: ..." - msgBody.originLine = line; - preOrigin = false; - } else if(self.KLUDGE_PREFIX === line.charAt(0)) { - addKludgeLine(line.slice(1)); - } else if(!sauceBuffers || _.startsWith(line, '\x1aSAUCE00')) { - sauceBuffers = sauceBuffers || buffers(); - sauceBuffers.push(new Buffer(line)); - } else { - msgBody.message.push(line); - } - } else { - if(_.startsWith(line, 'SEEN-BY:')) { - msgBody.seenBy.push(line.substring(line.indexOf(':') + 1).trim()); - } else if(self.KLUDGE_PREFIX === line.charAt(0)) { - addKludgeLine(line.slice(1)); - } - } - }); - - if(sauceBuffers) { - // :TODO: parse sauce -> sauce buffer. This needs changes to this method to return message & optional sauce - } - - cb(null, msgBody); - }; - - this.extractMessages = function(buffer, iterator, cb) { - assert(Buffer.isBuffer(buffer)); - assert(_.isFunction(iterator)); - - const NULL_TERM_BUFFER = new Buffer( [ 0 ] ); - - binary.stream(buffer).loop(function looper(end, vars) { - this - .word16lu('messageType') - .word16lu('origNode') - .word16lu('destNode') - .word16lu('origNet') - .word16lu('destNet') - .word8('attrFlags1') - .word8('attrFlags2') - .word16lu('cost') - .scan('modDateTime', NULL_TERM_BUFFER) - .scan('toUserName', NULL_TERM_BUFFER) - .scan('fromUserName', NULL_TERM_BUFFER) - .scan('subject', NULL_TERM_BUFFER) - .scan('message', NULL_TERM_BUFFER) - .tap(function tapped(msgData) { - if(!msgData.origNode) { - end(); - cb(null); - return; - } - - // buffer to string conversion - [ 'modDateTime', 'toUserName', 'fromUserName', 'subject', ].forEach(function field(f) { - msgData[f] = iconv.decode(msgData[f], 'CP437'); - }); - - self.parseFtnMessageBody(msgData.message, function msgBodyParsed(err, msgBody) { - // - // Now, create a Message object - // - var msg = new Message( { - // AREA FTN -> local conf/area occurs elsewhere - toUserName : msgData.toUserName, - fromUserName : msgData.fromUserName, - subject : msgData.subject, - message : msgBody.message.join('\n'), // :TODO: \r\n is better? - modTimestamp : ftn.getDateFromFtnDateTime(msgData.modDateTime), - meta : self.getMessageMeta(msgBody, msgData), - - - }); - - iterator(msg); - //self.emit('message', msg); // :TODO: Placeholder - }); - }); - }); - }; - - //this.getMessageHeaderBuffer = function(headerInfo) - - this.parseFtnMessages = function(buffer, cb) { - var nullTermBuf = new Buffer( [ 0 ] ); - var fidoMessages = []; - - binary.stream(buffer).loop(function looper(end, vars) { - this - .word16lu('messageType') - .word16lu('origNode') - .word16lu('destNode') - .word16lu('origNet') - .word16lu('destNet') - .word8('attrFlags1') - .word8('attrFlags2') - .word16lu('cost') - .scan('modDateTime', nullTermBuf) - .scan('toUserName', nullTermBuf) - .scan('fromUserName', nullTermBuf) - .scan('subject', nullTermBuf) - .scan('message', nullTermBuf) - .tap(function tapped(msgData) { - if(!msgData.origNode) { - end(); - cb(null, fidoMessages); - return; - } - - // buffer to string conversion - // :TODO: What is the real encoding here? - [ 'modDateTime', 'toUserName', 'fromUserName', 'subject', ].forEach(function field(f) { - msgData[f] = msgData[f].toString(); - }); - - self.parseFtnMessageBody(msgData.message, function msgBodyParsed(err, msgBody) { - msgData.message = msgBody; - fidoMessages.push(_.clone(msgData)); - }); - }); - }); - }; - - this.extractMesssagesFromPacketBuffer = function(packetBuffer, iterator, cb) { - assert(Buffer.isBuffer(packetBuffer)); - assert(_.isFunction(iterator)); - - async.waterfall( - [ - function parseHeader(callback) { - self.parseFtnPacketHeader(packetBuffer, function headerParsed(err, packetHeader) { - self.packetHeader = packetHeader; - callback(err); - }); - }, - function validateDesinationAddress(callback) { - self.localNetworkName = self.getNetworkNameForAddress(self.getPacketHeaderAddress()); - self.localNetworkName = 'AllowAnyNetworkForDebugging'; - callback(self.localNetworkName ? null : new Error('Packet not addressed do this system')); - }, - function extractEmbeddedMessages(callback) { - // note: packet header is 58 bytes in length - self.extractMessages( - packetBuffer.slice(58), iterator, function extracted(err) { - callback(err); - }); - } - ], - function complete(err) { - cb(err); - } - ); - }; - - this.loadMessagesFromPacketBuffer = function(packetBuffer, cb) { - async.waterfall( - [ - function parseHeader(callback) { - self.parseFtnPacketHeader(packetBuffer, function headerParsed(err, packetHeader) { - self.packetHeader = packetHeader; - callback(err); - }); - }, - function validateDesinationAddress(callback) { - self.localNetworkName = self.getNetworkNameForAddress(self.getPacketHeaderAddress()); - self.localNetworkName = 'AllowAnyNetworkForDebugging'; - callback(self.localNetworkName ? null : new Error('Packet not addressed do this system')); - }, - function parseMessages(callback) { - self.parseFtnMessages(packetBuffer.slice(58), function messagesParsed(err, fidoMessages) { - callback(err, fidoMessages); - }); - }, - function createMessageObjects(fidoMessages, callback) { - fidoMessages.forEach(function msg(fmsg) { - console.log(fmsg); - }); - } - ], - function complete(err) { - cb(err); - } - ); - }; -} - -//require('util').inherits(FTNMailPacket, MailPacket); - -FTNMailPacket.prototype.parse = function(path, cb) { - var self = this; - - async.waterfall( - [ - function readFromFile(callback) { - fs.readFile(path, function packetData(err, data) { - callback(err, data); - }); - }, - function extractMessages(data, callback) { - self.loadMessagesFromPacketBuffer(data, function extracted(err, messages) { - callback(err, messages); - }); - } - ], - function complete(err, messages) { - cb(err, messages); - } - ); -}; - -FTNMailPacket.prototype.read = function(pathOrBuffer, iterator, cb) { - var self = this; - - if(_.isString(pathOrBuffer)) { - async.waterfall( - [ - function readPacketFile(callback) { - fs.readFile(pathOrBuffer, function packetData(err, data) { - callback(err, data); - }); - }, - function extractMessages(data, callback) { - self.extractMesssagesFromPacketBuffer(data, iterator, callback); - } - ], - cb - ); - } else if(Buffer.isBuffer(pathOrBuffer)) { - - } -}; - -FTNMailPacket.prototype.write = function(messages, fileName, options) { - if(!_.isArray(messages)) { - messages = [ messages ]; - } - - - -}; - var ftnPacket = new FTNPacket(); var theHeader; var written = false; @@ -1023,67 +569,23 @@ ftnPacket.read( }); } + + let address = { + zone : 46, + net : 1, + node : 232, + domain : 'l33t.codes', + }; + msg.areaTag = 'agn_bbs'; + msg.messageId = 1234; + console.log(ftn.getMessageIdentifier(msg, address)); + console.log(ftn.getProductIdentifier()) + //console.log(ftn.getOrigin(address)) + console.log(ftn.parseAddress('46:1/232.4@l33t.codes')) + console.log(ftn.getUTCTimeZoneOffset()) } }, function completion(err) { console.log(err); } ); - -/* -var mailPacket = new FTNMailPacket( - { - nodeAddresses : { - fidoNet : { - zone : 46, - net : 1, - node : 140, - point : 0, - domain : '' - } - } - } -); - - -var didWrite = false; -mailPacket.read( - process.argv[2], - //'/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/mf/extracted/27000425.pkt', - function packetIter(msg) { - console.log(msg); - if(_.has(msg, 'meta.FtnProperty.ftn_area')) { - console.log('AREA: ' + msg.meta.FtnProperty.ftn_area); - } - - if(!didWrite) { - console.log(mailPacket.packetHeader); - console.log('-----------'); - - - didWrite = true; - - let outTest = fs.createWriteStream('/home/nuskooler/Downloads/ftnout/test1.pkt'); - let buffer = mailPacket.getPacketHeaderBuffer(mailPacket.packetHeader); - //mailPacket.write(buffer, msg.packetHeader); - outTest.write(buffer); - } - }, - function complete(err) { - console.log(err); - } -); -*/ -/* - Area Map - networkName: { - area_tag: conf_name:area_tag_name - ... - } -*/ - -/* -mailPacket.parse('/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007', function parsed(err, messages) { - console.log(err) -}); -*/ \ No newline at end of file diff --git a/core/ftn_util.js b/core/ftn_util.js index 41c8dfc6..94f716f2 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -11,16 +11,37 @@ var fs = require('fs'); var util = require('util'); var iconv = require('iconv-lite'); var moment = require('moment'); +var createHash = require('crypto').createHash; +var uuid = require('node-uuid'); +var os = require('os'); + +var packageJson = require('../package.json'); // :TODO: Remove "Ftn" from most of these -- it's implied in the module exports.stringFromFTN = stringFromFTN; exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; -exports.getFormattedFTNAddress = getFormattedFTNAddress; +exports.createMessageUuid = createMessageUuid; +exports.parseAddress = parseAddress; +exports.formatAddress = formatAddress; exports.getDateFromFtnDateTime = getDateFromFtnDateTime; exports.getDateTimeString = getDateTimeString; +exports.getMessageIdentifier = getMessageIdentifier; +exports.getProductIdentifier = getProductIdentifier; +exports.getUTCTimeZoneOffset = getUTCTimeZoneOffset; +exports.getOrigin = getOrigin; + exports.getQuotePrefix = getQuotePrefix; +// +// Namespace for RFC-4122 name based UUIDs generated from +// FTN kludges MSGID + AREA +// +const ENIGMA_FTN_MSGID_NAMESPACE = uuid.parse('a5c7ae11-420c-4469-a116-0e9a6d8d2654'); + +// Up to 5D FTN address RegExp +const ENIGMA_FTN_ADDRESS_REGEXP = /^([0-9]+):([0-9]+)(\/[0-9]+)?(\.[0-9]+)?(@[a-z0-9\-\.]+)?$/i; + // See list here: https://github.com/Mithgol/node-fidonet-jam // :TODO: proably move this elsewhere as a general method @@ -48,6 +69,7 @@ function stringToNullPaddedBuffer(s, bufLen) { // // Convert a FTN style DateTime string to a Date object // +// :TODO: Name the next couple methods better - for FTN *packets* function getDateFromFtnDateTime(dateTime) { // // Examples seen in the wild (Working): @@ -63,10 +85,10 @@ function getDateTimeString(m) { // // From http://ftsc.org/docs/fts-0001.016: // DateTime = (* a character string 20 characters long *) - // (* 01 Jan 86 02:34:56 *) - // DayOfMonth " " Month " " Year " " - // " " HH ":" MM ":" SS - // Null + // (* 01 Jan 86 02:34:56 *) + // DayOfMonth " " Month " " Year " " + // " " HH ":" MM ":" SS + // Null // // DayOfMonth = "01" | "02" | "03" | ... | "31" (* Fido 0 fills *) // Month = "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | @@ -83,42 +105,172 @@ function getDateTimeString(m) { return m.format('DD MMM YY HH:mm:ss'); } -function getFormattedFTNAddress(address, dimensions) { - //var addr = util.format('%d:%d', address.zone, address.net); - var addr = '{0}:{1}'.format(address.zone, address.net); - switch(dimensions) { - case 2 : - case '2D' : - // above - break; +function createMessageUuid(ftnMsgId, ftnArea) { + // + // v5 UUID generation code based on the work here: + // https://github.com/download13/uuidv5/blob/master/uuid.js + // + // Note: CrashMail uses MSGID + AREA, so we go with that as well: + // https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c + // + if(!Buffer.isBuffer(ftnMsgId)) { + ftnMsgId = iconv.encode(ftnMsgId, 'CP437'); + } - case 3 : - case '3D' : - addr += '/{0}'.format(address.node); - break; + ftnArea = ftnArea || ''; // AREA is optional + if(!Buffer.isBuffer(ftnArea)) { + ftnArea = iconv.encode(ftnArea, 'CP437'); + } + + const ns = new Buffer(ENIGMA_FTN_MSGID_NAMESPACE); - case 4 : - case '4D': - addr += '.{0}'.format(address.point || 0); // missing and 0 are equiv for point - break; + let digest = createHash('sha1').update( + Buffer.concat([ ns, ftnMsgId, ftnArea ])).digest(); - case 5 : - case '5D' : - if(address.domain) { - addr += '@{0}'.format(address.domain); - } - break; + let u = new Buffer(16); + + // bbbb - bb - bb - bb - bbbbbb + digest.copy(u, 0, 0, 4); // time_low + digest.copy(u, 4, 4, 6); // time_mid + digest.copy(u, 6, 6, 8); // time_hi_and_version + + u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101) + u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10 + u[9] = digest[9]; + + digest.copy(u, 10, 10, 16); + + return uuid.unparse(u); // to string +} + +function parseAddress(address) { + const m = ENIGMA_FTN_ADDRESS_REGEXP.exec(address); + + if(m) { + let addr = { + zone : parseInt(m[1]), + net : parseInt(m[2]), + }; + + // + // substr(1) on the following to remove the + // captured prefix + // + if(m[3]) { + addr.node = parseInt(m[3].substr(1)); + } + + if(m[4]) { + addr.point = parseInt(m[4].substr(1)); + } + + if(m[5]) { + addr.domain = m[5].substr(1); + } + + return addr; + } +} + +function formatAddress(address, dimensions) { + let addr = `${address.zone}:${address.net}`; + + // allow for e.g. '4D' or 5 + const dim = parseInt(dimensions.toString()[0]); + + if(dim >= 3) { + addr += `/${address.node}`; + } + + // missing & .0 are equiv for point + if(dim >= 4 && address.point) { + addr += `.${addresss.point}`; + } + + if(5 === dim && address.domain) { + addr += `@${address.domain.toLowerCase()}`; } return addr; } -function getFtnMessageSerialNumber(messageId) { - return ((Math.floor((Date.now() - Date.UTC(2015, 1, 1)) / 1000) + messageId)).toString(16); +function getMessageSerialNumber(message) { + return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + + message.messageId)).toString(16)).substr(-8); } +// +// Return a FTS-0009.001 compliant MSGID value given a message +// See http://ftsc.org/docs/fts-0009.001 +// +// "A MSGID line consists of the string "^AMSGID:" (where ^A is a +// control-A (hex 01) and the double-quotes are not part of the +// string), followed by a space, the address of the originating +// system, and a serial number unique to that message on the +// originating system, i.e.: +// +// ^AMSGID: origaddr serialno +// +// The originating address should be specified in a form that +// constitutes a valid return address for the originating network. +// If the originating address is enclosed in double-quotes, the +// entire string between the beginning and ending double-quotes is +// considered to be the orginating address. A double-quote character +// within a quoted address is represented by by two consecutive +// double-quote characters. The serial number may be any eight +// character hexadecimal number, as long as it is unique - no two +// messages from a given system may have the same serial number +// within a three years. The manner in which this serial number is +// generated is left to the implementor." +// +// +// Examples & Implementations +// +// Synchronet: .@ +// 2606.agora-agn_tst@46:1/142 19609217 +// +// Mystic: +// 46:3/102 46686263 +// +// ENiGMA½: .@<5dFtnAddress> +// +function getMessageIdentifier(message, address) { + return `${message.messageId}.${message.areaTag.toLowerCase()}@${formatAddress(address, '5D')} ${getMessageSerialNumber(message)}`; +} + +// +// Return a FSC-0046.005 Product Identifier or "PID" +// http://ftsc.org/docs/fsc-0046.005 +// +function getProductIdentifier() { + const version = packageJson.version + .replace(/\-/g, '.') + .replace(/alpha/,'a') + .replace(/beta/,'b'); + + const nodeVer = process.version.substr(1); // remove 'v' prefix + + return `ENiGMA1/2 ${version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; +} + +// +// Return a FSC-0030.001 compliant (http://ftsc.org/docs/fsc-0030.001) MESSAGE-ID +// +// +// +// :TODO: not implemented to spec at all yet :) function getFTNMessageID(messageId, areaId) { - return messageId + '.' + areaId + '@' + getFTNAddress() + ' ' + getFTNMessageSerialNumber(messageId) + return messageId + '.' + areaId + '@' + getFTNAddress() + ' ' + getMessageSerialNumber(messageId) +} + +// +// Return a FRL-1004 style time zone offset for a +// 'TZUTC' kludge line +// +// http://ftsc.org/docs/frl-1004.002 +// +function getUTCTimeZoneOffset() { + return moment().format('ZZ').replace(/\+/, ''); } // Get a FSC-0032 style quote prefixes @@ -127,25 +279,14 @@ function getQuotePrefix(name) { return ' ' + name[0].toUpperCase() + name[1].toLowerCase() + '> '; } - // -// Specs: -// * http://ftsc.org/docs/fts-0009.001 -// * -// -function getFtnMsgIdKludgeLine(origAddress, messageId) { - if(_.isObject(origAddress)) { - origAddress = getFormattedFTNAddress(origAddress, '5D'); - } +// Return a FTS-0004 Origin line +// http://ftsc.org/docs/fts-0004.001 +// +function getOrigin(address) { + const origin = _.has(Config.messageNetworks.originName) ? + Config.messageNetworks.originName : + Config.general.boardName; - return '\x01MSGID: ' + origAddress + ' ' + getFtnMessageSerialNumber(messageId); -} - - -function getFTNOriginLine() { - // - // Specs: - // http://ftsc.org/docs/fts-0004.001 - // - return ' * Origin: ' + Config.general.boardName + '(' + getFidoNetAddress() + ')'; + return ` * Origin: ${origin} (${formatAddress(address, '5D')})`; } diff --git a/core/message.js b/core/message.js index cdbfe67f..4e0bd0a9 100644 --- a/core/message.js +++ b/core/message.js @@ -17,7 +17,7 @@ function Message(options) { this.messageId = options.messageId || 0; // always generated @ persist this.areaTag = options.areaTag || Message.WellKnownAreaTags.Invalid; - this.uuid = uuid.v1(); + this.uuid = options.uuid || uuid.v1(); this.replyToMsgId = options.replyToMsgId || 0; this.toUserName = options.toUserName || ''; this.fromUserName = options.fromUserName || ''; diff --git a/core/msg_network_module.js b/core/msg_network_module.js deleted file mode 100644 index 8d4a9e0c..00000000 --- a/core/msg_network_module.js +++ /dev/null @@ -1,25 +0,0 @@ -/* jslint node: true */ -'use strict'; - -// ENiGMA½ -var PluginModule = require('./plugin_module.js').PluginModule; - -exports.MessageNetworkModule = MessageNetworkModule; - -function MessageNetworkModule() { - PluginModule.call(this); -} - -require('util').inherits(MessageNetworkModule, PluginModule); - -MessageNetworkModule.prototype.startup = function(cb) { - cb(null); -}; - -MessageNetworkModule.prototype.shutdown = function(cb) { - cb(null); -}; - -MessageNetworkModule.prototype.record = function(message, cb) { - cb(null); -}; \ No newline at end of file diff --git a/core/msg_networks/ftn_msg_network_module.js b/core/msg_networks/ftn_msg_network_module.js deleted file mode 100644 index f81aef9f..00000000 --- a/core/msg_networks/ftn_msg_network_module.js +++ /dev/null @@ -1,27 +0,0 @@ -/* jslint node: true */ -'use strict'; - -// ENiGMA½ -var MessageNetworkModule = require('./msg_network_module.js').MessageNetworkModule; - -function FTNMessageNetworkModule() { - MessageNetworkModule.call(this); -} - -require('util').inherits(FTNMessageNetworkModule, MessageNetworkModule); - -FTNMessageNetworkModule.prototype.startup = function(cb) { - cb(null); -}; - -FTNMessageNetworkModule.prototype.shutdown = function(cb) { - cb(null); -}; - -FTNMessageNetworkModule.prototype.record = function(message, cb) { - cb(null); - - // :TODO: should perhaps record in batches - e.g. start an event, record - // to temp location until time is hit or N achieved such that if multiple - // messages are being created a .FTN file is not made for each one -}; \ No newline at end of file diff --git a/core/msg_scan_toss_module.js b/core/msg_scan_toss_module.js new file mode 100644 index 00000000..e396f44d --- /dev/null +++ b/core/msg_scan_toss_module.js @@ -0,0 +1,25 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +var PluginModule = require('./plugin_module.js').PluginModule; + +exports.MessageScanTossModule = MessageScanTossModule; + +function MessageScanTossModule() { + PluginModule.call(this); +} + +require('util').inherits(MessageScanTossModule, PluginModule); + +MessageScanTossModule.prototype.startup = function(cb) { + cb(null); +}; + +MessageScanTossModule.prototype.shutdown = function(cb) { + cb(null); +}; + +MessageScanTossModule.prototype.record = function(message, cb) { + cb(null); +}; \ No newline at end of file diff --git a/core/sauce.js b/core/sauce.js index 6db85533..0dad1bc9 100644 --- a/core/sauce.js +++ b/core/sauce.js @@ -6,11 +6,11 @@ var iconv = require('iconv-lite'); exports.readSAUCE = readSAUCE; - const SAUCE_SIZE = 128; const SAUCE_ID = new Buffer([0x53, 0x41, 0x55, 0x43, 0x45]); // 'SAUCE' const COMNT_ID = new Buffer([0x43, 0x4f, 0x4d, 0x4e, 0x54]); // 'COMNT' +exports.SAUCE_SIZE = SAUCE_SIZE; // :TODO: SAUCE should be a class // - with getFontName() // - ...other methods @@ -19,6 +19,8 @@ const COMNT_ID = new Buffer([0x43, 0x4f, 0x4d, 0x4e, 0x54]); // 'COMNT' // See // http://www.acid.org/info/sauce/sauce.htm // +const SAUCE_VALID_DATA_TYPES = [0, 1, 2, 3, 4, 5, 6, 7, 8 ]; + function readSAUCE(data, cb) { if(data.length < SAUCE_SIZE) { cb(new Error('No SAUCE record present')); @@ -59,6 +61,11 @@ function readSAUCE(data, cb) { return; } + if(-1 === SAUCE_VALID_DATA_TYPES.indexOf(vars.dataType)) { + cb(new Error('Unsupported SAUCE DataType: ' + vars.dataType)); + return; + } + var sauce = { id : iconv.decode(vars.id, 'cp437'), version : iconv.decode(vars.version, 'cp437').trim(), diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js new file mode 100644 index 00000000..5b2283e5 --- /dev/null +++ b/core/scanner_tossers/ftn_bso.js @@ -0,0 +1,42 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +var MessageScanTossModule = require('../scan_toss_module.js').MessageScanTossModule; +var Config = require('../config.js').config; + +exports.moduleInfo = { + name : 'FTN', + desc : 'FidoNet Style Message Scanner/Tosser', + author : 'NuSkooler', +}; + +exports.getModule = FTNMessageScanTossModule; + +function FTNMessageScanTossModule() { + MessageScanTossModule.call(this); + + this.config = Config.scannerTossers.ftn_bso; + + +} + +require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); + +FTNMessageScanTossModule.prototype.startup = function(cb) { + cb(null); +}; + +FTNMessageScanTossModule.prototype.shutdown = function(cb) { + cb(null); +}; + +FTNMessageScanTossModule.prototype.record = function(message, cb) { + + + cb(null); + + // :TODO: should perhaps record in batches - e.g. start an event, record + // to temp location until time is hit or N achieved such that if multiple + // messages are being created a .FTN file is not made for each one +}; diff --git a/core/theme.js b/core/theme.js index f23c6481..5fd5db75 100644 --- a/core/theme.js +++ b/core/theme.js @@ -523,7 +523,8 @@ function displayThemedPause(options, cb) { if(options.clearPrompt) { if(artInfo.startRow && artInfo.height) { options.client.term.rawWrite(ansi.goto(artInfo.startRow, 1)); - // :TODO: This will not work with NetRunner: + + // Note: Does not work properly in NetRunner < 2.0b17: options.client.term.rawWrite(ansi.deleteLine(artInfo.height)); } else { options.client.term.rawWrite(ansi.eraseLine(1)) From 7b5ab029f978117a495cd25c1515ea55d6438515 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 10 Feb 2016 22:24:46 -0700 Subject: [PATCH 03/27] Many updates to read/write of packets of diff versions --- core/ftn_mail_packet.js | 191 ++++++++++++++++++++++++++++------------ 1 file changed, 133 insertions(+), 58 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 8b69228d..361d24dc 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -18,11 +18,10 @@ var moment = require('moment'); /* :TODO: things - * Read/detect packet types: 2, 2.2, and 2+ - * Write packet types: 2, 2.2, and 2+ * Test SAUCE ignore/extraction * FSP-1010 for netmail (see SBBS) - + * Syncronet apparently uses odd origin lines + * Origin lines starting with "#" instead of "*" ? */ @@ -39,7 +38,7 @@ const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; // // Read/Write FTN packets with support for the following formats: // -// * Type 1 FTS-0001 @ http://ftsc.org/docs/fts-0001.016 (Obsolete) +// * Type 2 FTS-0001 @ http://ftsc.org/docs/fts-0001.016 (Obsolete) // * Type 2.2 FSC-0045 @ http://ftsc.org/docs/fsc-0045.001 // * Type 2+ FSC-0039 and FSC-0048 @ http://ftsc.org/docs/fsc-0039.004 // and http://ftsc.org/docs/fsc-0048.002 @@ -81,15 +80,18 @@ function FTNPacket() { .buffer('password', 8) // null padded C style string .word16lu('origZone') .word16lu('destZone') - // Additions in FSC-0048.002 follow... + // + // The following is "filler" in FTS-0001, specifics in + // FSC-0045 and FSC-0048 + // .word16lu('auxNet') - .word16lu('capWordA') + .word16lu('capWordValidate') .word8('prodCodeHi') .word8('prodRevHi') - .word16lu('capWordB') + .word16lu('capWord') .word16lu('origZone2') .word16lu('destZone2') - .word16lu('originPoint') + .word16lu('origPoint') .word16lu('destPoint') .word32lu('prodData') .tap(packetHeader => { @@ -104,24 +106,38 @@ function FTNPacket() { // // What kind of packet do we really have here? // + // :TODO: adjust values based on version discoverd if(FTN_PACKET_BAUD_TYPE_2_2 === packetHeader.baud) { packetHeader.packetVersion = '2.2'; + + // See FSC-0045 + packetHeader.origPoint = packetHeader.year; + packetHeader.destPoint = packetHeader.month; + + packetHeader.destDomain = packetHeader.origZone2; + packetHeader.origDomain = packetHeader.auxNet; } else { // // See heuristics described in FSC-0048, "Receiving Type-2+ bundles" // - const capWordASwapped = - ((packetHeader.capWordA & 0xff) << 8) | - ((packetHeader.capWordA >> 8) & 0xff); + const capWordValidateSwapped = + ((packetHeader.capWordValidate & 0xff) << 8) | + ((packetHeader.capWordValidate >> 8) & 0xff); - if(capWordASwapped === packetHeader.capWordB && - 0 != packetHeader.capWordB && - packetHeader.capWordB & 0x0001) + if(capWordValidateSwapped === packetHeader.capWord && + 0 != packetHeader.capWord && + packetHeader.capWord & 0x0001) { packetHeader.packetVersion = '2+'; + + // See FSC-0048 + if(-1 === packetHeader.origNet) { + packetHeader.origNet = packetHeader.auxNet; + } } else { packetHeader.packetVersion = '2'; - packetHeader.point + + // :TODO: should fill bytes be 0? } } @@ -135,41 +151,74 @@ function FTNPacket() { }); }; - this.writePacketHeader = function(headerInfo, ws) { + this.writePacketHeader = function(packetHeader, ws) { let buffer = new Buffer(FTN_PACKET_HEADER_SIZE); - buffer.writeUInt16LE(headerInfo.origNode, 0); - buffer.writeUInt16LE(headerInfo.destNode, 2); - buffer.writeUInt16LE(headerInfo.created.year(), 4); - buffer.writeUInt16LE(headerInfo.created.month(), 6); - buffer.writeUInt16LE(headerInfo.created.date(), 8); - buffer.writeUInt16LE(headerInfo.created.hour(), 10); - buffer.writeUInt16LE(headerInfo.created.minute(), 12); - buffer.writeUInt16LE(headerInfo.created.second(), 14); - buffer.writeUInt16LE(headerInfo.baud, 16); + // :TODO: write 2, 2.2, or 2+ packet based on packetHeader.packetVersion (def=2+) + + buffer.writeUInt16LE(packetHeader.origNode, 0); + buffer.writeUInt16LE(packetHeader.destNode, 2); + buffer.writeUInt16LE(packetHeader.created.year(), 4); + buffer.writeUInt16LE(packetHeader.created.month(), 6); + buffer.writeUInt16LE(packetHeader.created.date(), 8); + buffer.writeUInt16LE(packetHeader.created.hour(), 10); + buffer.writeUInt16LE(packetHeader.created.minute(), 12); + buffer.writeUInt16LE(packetHeader.created.second(), 14); + buffer.writeUInt16LE(packetHeader.baud, 16); buffer.writeUInt16LE(FTN_PACKET_HEADER_TYPE, 18); - buffer.writeUInt16LE(headerInfo.origNet, 20); - buffer.writeUInt16LE(headerInfo.destNet, 22); - buffer.writeUInt8(headerInfo.prodCodeLo, 24); - buffer.writeUInt8(headerInfo.prodRevHi, 25); + buffer.writeUInt16LE(packetHeader.origNet, 20); + buffer.writeUInt16LE(packetHeader.destNet, 22); + buffer.writeUInt8(packetHeader.prodCodeLo, 24); + buffer.writeUInt8(packetHeader.prodRevHi, 25); - const pass = ftn.stringToNullPaddedBuffer(headerInfo.password, 8); + const pass = ftn.stringToNullPaddedBuffer(packetHeader.password, 8); pass.copy(buffer, 26); - buffer.writeUInt16LE(headerInfo.origZone, 34); - buffer.writeUInt16LE(headerInfo.destZone, 36); + buffer.writeUInt16LE(packetHeader.origZone, 34); + buffer.writeUInt16LE(packetHeader.destZone, 36); + + switch(packetHeader.packetType) { + case '2' : + // filler... + packetHeader.auxNet = 0; + packetHeader.capWordValidate = 0; + packetHeader.prodCodeHi = 0; + packetHeader.prodRevLo = 0; + packetHeader.capWord = 0; + packetHeader.origZone2 = 0; + packetHeader.destZone2 = 0; + packetHeader.origPoint = 0; + packetHeader.destPoint = 0; + break; + + case '2.2' : + packetHeader.day = 0; + packetHeader.hour = 0; + packetHeader.minute = 0; + packetHeader.second = 0; + break; + + case '2+' : + break; + + } + + buffer.writeUInt16LE(packetHeader.auxNet, 38); + buffer.writeUInt16LE(packetHeader.capWordValidate, 40); + buffer.writeUInt8(packetHeader.prodCodeHi, 42); + buffer.writeUInt8(packetHeader.prodRevLo, 43); + buffer.writeUInt16LE(packetHeader.capWord, 44); + buffer.writeUInt16LE(packetHeader.origZone2, 46); + buffer.writeUInt16LE(packetHeader.destZone2, 48); + buffer.writeUInt16LE(packetHeader.origPoint, 50); + buffer.writeUInt16LE(packetHeader.destPoint, 52); - // FSC-0048.002 additions... - buffer.writeUInt16LE(headerInfo.auxNet, 38); - buffer.writeUInt16LE(headerInfo.capWordA, 40); - buffer.writeUInt8(headerInfo.prodCodeHi, 42); - buffer.writeUInt8(headerInfo.prodRevLo, 43); - buffer.writeUInt16LE(headerInfo.capWordB, 44); - buffer.writeUInt16LE(headerInfo.origZone2, 46); - buffer.writeUInt16LE(headerInfo.destZone2, 48); - buffer.writeUInt16LE(headerInfo.origPoint, 50); - buffer.writeUInt16LE(headerInfo.destPoint, 52); - buffer.writeUInt32LE(headerInfo.prodData, 54); + // Store in "ENiG" in prodData unless we already have something useful + if(0 === packetHeader.prodData) { + packetHeader.prodData = 0x47694e45; + } + + buffer.writeUInt32LE(packetHeader.prodData, 54); ws.write(buffer); }; @@ -292,6 +341,8 @@ function FTNPacket() { this.parsePacketMessages = function(messagesBuffer, iterator, cb) { const NULL_TERM_BUFFER = new Buffer( [ 0 ] ); + + var count = 0; binary.stream(messagesBuffer).loop(function looper(end, vars) { // @@ -315,16 +366,18 @@ function FTNPacket() { .tap(function tapped(msgData) { if(!msgData.ftn_orig_node) { // end marker -- no more messages - end(); - cb(null); + end(); return; } if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) { end(); + // :TODO: This is probably a bug if we hit a bad message after at leats one iterate cb(new Error('Unsupported message type: ' + msgData.messageType)); return; } + + ++count; // // Convert null terminated arrays to strings @@ -386,6 +439,11 @@ function FTNPacket() { } iterator('message', msg); + + --count; + if(0 === count) { + cb(null); + } }) }); }); @@ -439,7 +497,7 @@ function FTNPacket() { a = [ a ]; } a.forEach(v => { - msgBody += `${k}: ${v}\n`; + msgBody += `${k}: ${v}\r`; }); } } @@ -450,13 +508,13 @@ function FTNPacket() { // Should be first line in a message // if(message.meta.FtnProperty.ftn_area) { - msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\n`; + msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\r`; // note: no ^A (0x01) } Object.keys(message.meta.FtnKludge).forEach(k => { // we want PATH to be last if('PATH' !== k) { - appendMeta(k, message.meta.FtnKludge[k]); + appendMeta(`\x01${k}`, message.meta.FtnKludge[k]); } }); @@ -477,8 +535,8 @@ function FTNPacket() { // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 // SEEN-BY and PATH should be the last lines of a message // - appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); - appendMeta('PATH', message.meta.FtnKludge['PATH']); + appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); // note: no ^A (0x01) + appendMeta('\x01PATH', message.meta.FtnKludge['PATH']); ws.write(iconv.encode(msgBody + '\0', 'CP437')); }; @@ -501,7 +559,7 @@ function FTNPacket() { callback); } ], - cb + cb // complete ); }; } @@ -522,16 +580,21 @@ FTNPacket.prototype.read = function(pathOrBuffer, iterator, cb) { } }, function parseBuffer(callback) { - self.parsePacketBuffer(pathOrBuffer, iterator, callback); + self.parsePacketBuffer(pathOrBuffer, iterator, err => { + callback(err); + }); } ], - cb // completion callback + function complete(err) { + cb(err); + } ); }; -FTNPacket.prototype.write = function(path, headerInfo, messages, cb) { - headerInfo.created = headerInfo.created || moment(); - headerInfo.baud = headerInfo.baud || 0; +FTNPacket.prototype.write = function(path, packetHeader, messages, cb) { + packetHeader.packetType = packetHeader.packetType || '2+'; + packetHeader.created = packetHeader.created || moment(); + packetHeader.baud = packetHeader.baud || 0; // :TODO: Other defaults? if(!_.isArray(messages)) { @@ -539,17 +602,20 @@ FTNPacket.prototype.write = function(path, headerInfo, messages, cb) { } let ws = fs.createWriteStream(path); - this.writePacketHeader(headerInfo, ws); + this.writePacketHeader(packetHeader, ws); messages.forEach(msg => { this.writeMessage(msg, ws); }); + + ws.write(new Buffer( [ 0 ] )); // final extra null term }; var ftnPacket = new FTNPacket(); var theHeader; var written = false; +let messagesToWrite = []; ftnPacket.read( process.argv[2], function iterator(dataType, data) { @@ -560,6 +626,9 @@ ftnPacket.read( const msg = data; console.log(msg); + messagesToWrite.push(msg); + + /* if(!written) { written = true; @@ -568,7 +637,7 @@ ftnPacket.read( }); - } + }*/ let address = { zone : 46, @@ -586,6 +655,12 @@ ftnPacket.read( } }, function completion(err) { + console.log('complete!') console.log(err); + + + console.log(messagesToWrite.length) + ftnPacket.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messagesToWrite, err => { + }); } ); From 13d5c4d8f4574ce685ea1afbfb878f9cf1765470 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 15 Feb 2016 17:56:05 -0700 Subject: [PATCH 04/27] * New Address class for FTN addresses + experiment with ES6 classes * Move a lot of address functionality/parsing/etc. to Address * WIP on ftn_bso scan/tosser * PATH and SEEN-BY creation, parsing, etc. --- core/ftn_address.js | 169 ++++++++++++++++++++ core/ftn_mail_packet.js | 273 ++++++++++++++++++++++++++------ core/ftn_util.js | 203 +++++++++++++++--------- core/message.js | 5 +- core/scanner_tossers/ftn_bso.js | 63 +++++++- core/string_util.js | 31 +++- 6 files changed, 605 insertions(+), 139 deletions(-) create mode 100644 core/ftn_address.js diff --git a/core/ftn_address.js b/core/ftn_address.js new file mode 100644 index 00000000..e6dad9e9 --- /dev/null +++ b/core/ftn_address.js @@ -0,0 +1,169 @@ +/* jslint node: true */ +'use strict'; + +let _ = require('lodash'); + +const FTN_ADDRESS_REGEXP = /^([0-9]+:)?([0-9]+)(\/[0-9]+)?(\.[0-9]+)?(@[a-z0-9\-\.]+)?$/i; +const FTN_PATTERN_REGEXP = /^([0-9\*]+:)?([0-9\*]+)(\/[0-9\*]+)?(\.[0-9\*]+)?(@[a-z0-9\-\.\*]+)?$/i; + +module.exports = class Address { + constructor(addr) { + if(addr) { + if(_.isObject(addr)) { + Object.assign(this, addr); + } else if(_.isString(addr)) { + const temp = Address.fromString(addr); + if(temp) { + Object.assign(this, temp); + } + } + } + } + + isEqual(other) { + if(_.isString(other)) { + other = Address.fromString(other); + } + + return ( + this.net === other.net && + this.node === other.node && + this.zone === other.zone && + this.point === other.point && + this.domain === other.domain + ); + } + + isMatch(pattern) { + const m = FTN_PATTERN_REGEXP.exec(pattern); + if(m) { + let addr = { }; + + if(m[1]) { + addr.zone = m[1].slice(0, -1) + if('*' !== addr.zone) { + addr.zone = parseInt(addr.zone); + } + } else { + addr.zone = '*'; + } + + if(m[2]) { + addr.net = m[2]; + if('*' !== addr.net) { + addr.net = parseInt(addr.net); + } + } else { + addr.net = '*'; + } + + if(m[3]) { + addr.node = m[3].substr(1); + if('*' !== addr.node) { + addr.node = parseInt(addr.node); + } + } else { + addr.node = '*'; + } + + if(m[4]) { + addr.point = m[4].substr(1); + if('*' !== addr.point) { + addr.point = parseInt(addr.point); + } + } else { + addr.point = '*'; + } + + if(m[5]) { + addr.domain = m[5].substr(1); + } else { + addr.domain = '*'; + } + + return ( + ('*' === addr.net || this.net === addr.net) && + ('*' === addr.node || this.node === addr.node) && + ('*' === addr.zone || this.zone === addr.zone) && + ('*' === addr.point || this.point === addr.point) && + ('*' === addr.domain || this.domain === addr.domain) + ); + } + + return false; + } + + static fromString(addrStr) { + const m = FTN_ADDRESS_REGEXP.exec(addrStr); + + if(m) { + // start with a 2D + let addr = { + net : parseInt(m[2]), + node : parseInt(m[3].substr(1)), + }; + + // 3D: Addition of zone if present + if(m[1]) { + addr.zone = parseInt(m[1].slice(0, -1)); + } + + // 4D if optional point is present + if(m[4]) { + addr.point = parseInt(m[4].substr(1)); + } + + // 5D with @domain + if(m[5]) { + addr.domain = m[5].substr(1); + } + + return new Address(addr); + } + } + + toString(dimensions) { + dimensions = dimensions || '5D'; + + let addrStr = `${this.zone}:${this.net}`; + + // allow for e.g. '4D' or 5 + const dim = parseInt(dimensions.toString()[0]); + + if(dim >= 3) { + addrStr += `/${this.node}`; + } + + // missing & .0 are equiv for point + if(dim >= 4 && this.point) { + addrStr += `.${this.point}`; + } + + if(5 === dim && this.domain) { + addrStr += `@${this.domain.toLowerCase()}`; + } + + return addrStr; + } + + static getComparator() { + return function(left, right) { + let c = (left.zone || 0) - (right.zone || 0); + if(0 !== c) { + return c; + } + + c = (left.net || 0) - (right.net || 0); + if(0 !== c) { + return c; + } + + c = (left.node || 0) - (right.node || 0); + if(0 !== c) { + return c; + } + + return (left.domain || '').localeCompare(right.domain || ''); + } + } +} diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 361d24dc..51215c5d 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -2,19 +2,24 @@ 'use strict'; //var MailPacket = require('./mail_packet.js'); -var ftn = require('./ftn_util.js'); -var Message = require('./message.js'); -var sauce = require('./sauce.js'); +let ftn = require('./ftn_util.js'); +let Message = require('./message.js'); +let sauce = require('./sauce.js'); +let Address = require('./ftn_address.js'); +let strUtil = require('./string_util.js'); -var _ = require('lodash'); -var assert = require('assert'); -var binary = require('binary'); -var fs = require('fs'); -var util = require('util'); -var async = require('async'); -var iconv = require('iconv-lite'); -var buffers = require('buffers'); -var moment = require('moment'); +let _ = require('lodash'); +let assert = require('assert'); +let binary = require('binary'); +let fs = require('fs'); +let util = require('util'); +let async = require('async'); +let iconv = require('iconv-lite'); +let buffers = require('buffers'); +let moment = require('moment'); + +exports.PacketHeader = PacketHeader; +exports.Packet = Packet; /* :TODO: things @@ -35,6 +40,88 @@ const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; +function PacketHeader(options) { +} + +PacketHeader.prototype.init = function(origAddr, destAddr, created, version) { + version = version || '2+'; + + const EMPTY_ADDRESS = { + node : 0, + net : 0, + zone : 0, + point : 0, + }; + + this.packetVersion = version; + + this.setOrigAddress(origAddr || EMPTY_ADDRESS); + this.setDestAddress(destAddr || EMPTY_ADDRESS); + this.setCreated(created || moment()); + + // uncommon to set the following explicitly + this.prodCodeLo = 0xfe; // http://ftsc.org/docs/fta-1005.003 + this.prodRevLo = 0; + this.baud = 0; + this.packetType = FTN_PACKET_HEADER_TYPE; + this.password = ''; + this.prodData = 0x47694e45; // "ENiG" + + this.capWord = 0x0001; + this.capWordValidate = ((this.capWord & 0xff) << 8) | ((this.capWord >> 8) & 0xff); + + return this; +}; + +PacketHeader.prototype.setOrigAddress = function(address) { + this.origNode = address.node; + + // See FSC-48 + if(address.point) { + this.origNet = -1; + this.auxNet = address.net; + } else { + this.origNet = address.net; + this.auxNet = 0; + } + + this.origZone = address.zone; + this.origZone2 = address.zone; + this.origPoint = address.point || 0; + + return this; +}; + +PacketHeader.prototype.setDestAddress = function(address) { + this.destNode = address.node; + this.destNet = address.net; + this.destZone = address.zone; + this.destZone2 = address.zone; + this.destPoint = address.point || 0; + + return this; +}; + +PacketHeader.prototype.setCreated = function(created) { + if(!moment.isMoment(created)) { + created = moment(created); + } + + this.year = created.year(); + this.month = created.month(); + this.day = created.day(); + this.hour = created.hour(); + this.minute = created.minute(); + this.second = created.second(); + + return this; +}; + +PacketHeader.prototype.setPassword = function(password) { + this.password = password.substr(0, 8); +} + + // // Read/Write FTN packets with support for the following formats: // @@ -47,7 +134,7 @@ const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; // * Writeup on differences between type 2, 2.2, and 2+: // http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt // -function FTNPacket() { +function Packet() { var self = this; @@ -96,7 +183,8 @@ function FTNPacket() { .word32lu('prodData') .tap(packetHeader => { // Convert password from NULL padded array to string - packetHeader.password = ftn.stringFromFTN(packetHeader.password); + //packetHeader.password = ftn.stringFromFTN(packetHeader.password); + packetHeader.password = strUtil.stringFromNullTermBuffer(packetHeader.password, 'CP437'); if(FTN_PACKET_HEADER_TYPE !== packetHeader.packetType) { cb(new Error('Unsupported header type: ' + packetHeader.packetType)); @@ -106,7 +194,7 @@ function FTNPacket() { // // What kind of packet do we really have here? // - // :TODO: adjust values based on version discoverd + // :TODO: adjust values based on version discovered if(FTN_PACKET_BAUD_TYPE_2_2 === packetHeader.baud) { packetHeader.packetVersion = '2.2'; @@ -147,15 +235,16 @@ function FTNPacket() { // packetHeader.created = moment(packetHeader); - cb(null, packetHeader); + let ph = new PacketHeader(); + _.assign(ph, packetHeader); + + cb(null, ph); }); }; this.writePacketHeader = function(packetHeader, ws) { let buffer = new Buffer(FTN_PACKET_HEADER_SIZE); - // :TODO: write 2, 2.2, or 2+ packet based on packetHeader.packetVersion (def=2+) - buffer.writeUInt16LE(packetHeader.origNode, 0); buffer.writeUInt16LE(packetHeader.destNode, 2); buffer.writeUInt16LE(packetHeader.created.year(), 4); @@ -177,7 +266,8 @@ function FTNPacket() { buffer.writeUInt16LE(packetHeader.origZone, 34); buffer.writeUInt16LE(packetHeader.destZone, 36); - switch(packetHeader.packetType) { + // :TODO: update header information appropriately for 2 and 2.2 + switch(packetHeader.packetVersion) { case '2' : // filler... packetHeader.auxNet = 0; @@ -196,9 +286,20 @@ function FTNPacket() { packetHeader.hour = 0; packetHeader.minute = 0; packetHeader.second = 0; + + // :TODO: copy over fields from 2+ -> overriden fields here! + + packetHeader.baud = FTN_PACKET_BAUD_TYPE_2_2; break; case '2+' : + const capWordValidateSwapped = + ((packetHeader.capWordValidate & 0xff) << 8) | + ((packetHeader.capWordValidate >> 8) & 0xff); + + packetHeader.capWordValidate = capWordValidateSwapped; + + // :TODO: set header appropriate if point break; } @@ -355,8 +456,9 @@ function FTNPacket() { .word16lu('ftn_dest_node') .word16lu('ftn_orig_network') .word16lu('ftn_dest_network') - .word8('ftn_attr_flags1') - .word8('ftn_attr_flags2') + .word16lu('ftn_attr_flags') + //.word8('ftn_attr_flags1') + //.word8('ftn_attr_flags2') .word16lu('ftn_cost') .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max @@ -404,8 +506,9 @@ function FTNPacket() { msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node; msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; - msg.meta.FtnProperty.ftn_attr_flags1 = msgData.ftn_attr_flags1; - msg.meta.FtnProperty.ftn_attr_flags2 = msgData.ftn_attr_flags2; + msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; + //msg.meta.FtnProperty.ftn_attr_flags1 = msgData.ftn_attr_flags1; + //msg.meta.FtnProperty.ftn_attr_flags2 = msgData.ftn_attr_flags2; msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; self.processMessageBody(msgData.message, function processed(messageBodyData) { @@ -457,8 +560,9 @@ function FTNPacket() { basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); - basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags1, 10); - basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags2, 11); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); + //basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags1, 10); + //basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags2, 11); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); @@ -522,20 +626,25 @@ function FTNPacket() { // // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 - // Origin line should be near the bottom of a message - // - appendMeta('', message.meta.FtnProperty.ftn_tear_line); - - // // Tear line should be near the bottom of a message // - appendMeta('', message.meta.FtnProperty.ftn_origin); + if(message.meta.FtnProperty.ftn_tear_line) { + msgBody += `${message.meta.FtnProperty.ftn_tear_line}\r`; + } + + // + // Origin line should be near the bottom of a message + // + if(message.meta.FtnProperty.ftn_origin) { + msgBody += `${message.meta.FtnProperty.ftn_origin}\r`; + } // // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 // SEEN-BY and PATH should be the last lines of a message // appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); // note: no ^A (0x01) + appendMeta('\x01PATH', message.meta.FtnKludge['PATH']); ws.write(iconv.encode(msgBody + '\0', 'CP437')); @@ -564,7 +673,31 @@ function FTNPacket() { }; } -FTNPacket.prototype.read = function(pathOrBuffer, iterator, cb) { +// +// Message attributes defined in FTS-0001.016 +// http://ftsc.org/docs/fts-0001.016 +// +Packet.Attribute = { + Private : 0x0001, + Crash : 0x0002, + Received : 0x0004, + Sent : 0x0008, + FileAttached : 0x0010, + InTransit : 0x0020, + Orphan : 0x0040, + KillSent : 0x0080, + Local : 0x0100, + Hold : 0x0200, + Reserved0 : 0x0400, + FileRequest : 0x0800, + ReturnReceiptRequest : 0x1000, + ReturnReceipt : 0x2000, + AuditRequest : 0x4000, + FileUpdateRequest : 0x8000, +}; +Object.freeze(Packet.Attribute); + +Packet.prototype.read = function(pathOrBuffer, iterator, cb) { var self = this; async.series( @@ -591,32 +724,67 @@ FTNPacket.prototype.read = function(pathOrBuffer, iterator, cb) { ); }; -FTNPacket.prototype.write = function(path, packetHeader, messages, cb) { - packetHeader.packetType = packetHeader.packetType || '2+'; - packetHeader.created = packetHeader.created || moment(); - packetHeader.baud = packetHeader.baud || 0; - // :TODO: Other defaults? - - if(!_.isArray(messages)) { - messages = [ messages ] ; +Packet.prototype.writeStream = function(ws, messages, options) { + if(!_.isBoolean(options.terminatePacket)) { + options.terminatePacket = true; + } + + if(_.isObject(options.packetHeader)) { + /* + let packetHeader = options.packetHeader; + + // default header + packetHeader.packetVersion = packetHeader.packetVersion || '2+'; + packetHeader.created = packetHeader.created || moment(); + packetHeader.baud = packetHeader.baud || 0; + */ + this.writePacketHeader(options.packetHeader, ws); } - - let ws = fs.createWriteStream(path); - this.writePacketHeader(packetHeader, ws); messages.forEach(msg => { this.writeMessage(msg, ws); }); - - ws.write(new Buffer( [ 0 ] )); // final extra null term + + if(true === options.terminatePacket) { + ws.write(new Buffer( [ 0 ] )); // final extra null term + } +} + +Packet.prototype.write = function(path, packetHeader, messages) { + if(!_.isArray(messages)) { + messages = [ messages ]; + } + + this.writeStream( + fs.createWriteStream(path), // :TODO: specify mode/etc. + messages, + { packetHeader : packetHeader, terminatePacket : true } + ); }; -var ftnPacket = new FTNPacket(); +const LOCAL_ADDRESS = { + zone : 46, + net : 1, + node : 232, + domain : 'l33t.codes', +}; + +const REMOTE_ADDRESS = { + zone : 1, + net : 2, + node : 218, +}; + +var packetHeader1 = new PacketHeader(); +packetHeader1.init(LOCAL_ADDRESS, REMOTE_ADDRESS); +console.log(packetHeader1); + +var packet = new Packet(); var theHeader; var written = false; let messagesToWrite = []; -ftnPacket.read( +packet.read( process.argv[2], function iterator(dataType, data) { if('header' === dataType) { @@ -633,7 +801,7 @@ ftnPacket.read( written = true; let messages = [ msg ]; - ftnPacket.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messages, err => { + Packet.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messages, err => { }); @@ -650,8 +818,13 @@ ftnPacket.read( console.log(ftn.getMessageIdentifier(msg, address)); console.log(ftn.getProductIdentifier()) //console.log(ftn.getOrigin(address)) - console.log(ftn.parseAddress('46:1/232.4@l33t.codes')) + //console.log(ftn.parseAddress('46:1/232.4@l33t.codes')) + console.log(Address.fromString('46:1/232.4@l33t.codes')); console.log(ftn.getUTCTimeZoneOffset()) + console.log(ftn.getUpdatedSeenByEntries( + msg.meta.FtnProperty.ftn_seen_by, '1/107 4/22 4/25 4/10')); + console.log(ftn.getUpdatedPathEntries( + msg.meta.FtnKludge['PATH'], '1:365/50')) } }, function completion(err) { @@ -660,7 +833,7 @@ ftnPacket.read( console.log(messagesToWrite.length) - ftnPacket.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messagesToWrite, err => { + packet.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messagesToWrite, err => { }); } ); diff --git a/core/ftn_util.js b/core/ftn_util.js index 94f716f2..a5cec2a2 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -2,7 +2,7 @@ 'use strict'; var Config = require('./config.js').config; - +var Address = require('./ftn_address.js'); var _ = require('lodash'); var assert = require('assert'); @@ -18,20 +18,22 @@ var os = require('os'); var packageJson = require('../package.json'); // :TODO: Remove "Ftn" from most of these -- it's implied in the module -exports.stringFromFTN = stringFromFTN; exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; exports.createMessageUuid = createMessageUuid; -exports.parseAddress = parseAddress; -exports.formatAddress = formatAddress; -exports.getDateFromFtnDateTime = getDateFromFtnDateTime; -exports.getDateTimeString = getDateTimeString; +exports.getDateFromFtnDateTime = getDateFromFtnDateTime; +exports.getDateTimeString = getDateTimeString; -exports.getMessageIdentifier = getMessageIdentifier; -exports.getProductIdentifier = getProductIdentifier; -exports.getUTCTimeZoneOffset = getUTCTimeZoneOffset; -exports.getOrigin = getOrigin; +exports.getMessageIdentifier = getMessageIdentifier; +exports.getProductIdentifier = getProductIdentifier; +exports.getUTCTimeZoneOffset = getUTCTimeZoneOffset; +exports.getOrigin = getOrigin; +exports.getTearLine = getTearLine; +exports.getAbbreviatedNetNodeList = getAbbreviatedNetNodeList; +exports.parseAbbreviatedNetNodeList = parseAbbreviatedNetNodeList; +exports.getUpdatedSeenByEntries = getUpdatedSeenByEntries; +exports.getUpdatedPathEntries = getUpdatedPathEntries; -exports.getQuotePrefix = getQuotePrefix; +exports.getQuotePrefix = getQuotePrefix; // // Namespace for RFC-4122 name based UUIDs generated from @@ -40,23 +42,10 @@ exports.getQuotePrefix = getQuotePrefix; const ENIGMA_FTN_MSGID_NAMESPACE = uuid.parse('a5c7ae11-420c-4469-a116-0e9a6d8d2654'); // Up to 5D FTN address RegExp -const ENIGMA_FTN_ADDRESS_REGEXP = /^([0-9]+):([0-9]+)(\/[0-9]+)?(\.[0-9]+)?(@[a-z0-9\-\.]+)?$/i; +const ENIGMA_FTN_ADDRESS_REGEXP = /^([0-9]+:)?([0-9]+)(\/[0-9]+)?(\.[0-9]+)?(@[a-z0-9\-\.]+)?$/i; // See list here: https://github.com/Mithgol/node-fidonet-jam -// :TODO: proably move this elsewhere as a general method -function stringFromFTN(buf, encoding) { - var nullPos = buf.length; - for(var i = 0; i < buf.length; ++i) { - if(0x00 === buf[i]) { - nullPos = i; - break; - } - } - - return iconv.decode(buf.slice(0, nullPos), encoding || 'utf-8'); -} - function stringToNullPaddedBuffer(s, bufLen) { let buffer = new Buffer(bufLen).fill(0x00); let enc = iconv.encode(s, 'CP437').slice(0, bufLen); @@ -143,57 +132,6 @@ function createMessageUuid(ftnMsgId, ftnArea) { return uuid.unparse(u); // to string } -function parseAddress(address) { - const m = ENIGMA_FTN_ADDRESS_REGEXP.exec(address); - - if(m) { - let addr = { - zone : parseInt(m[1]), - net : parseInt(m[2]), - }; - - // - // substr(1) on the following to remove the - // captured prefix - // - if(m[3]) { - addr.node = parseInt(m[3].substr(1)); - } - - if(m[4]) { - addr.point = parseInt(m[4].substr(1)); - } - - if(m[5]) { - addr.domain = m[5].substr(1); - } - - return addr; - } -} - -function formatAddress(address, dimensions) { - let addr = `${address.zone}:${address.net}`; - - // allow for e.g. '4D' or 5 - const dim = parseInt(dimensions.toString()[0]); - - if(dim >= 3) { - addr += `/${address.node}`; - } - - // missing & .0 are equiv for point - if(dim >= 4 && address.point) { - addr += `.${addresss.point}`; - } - - if(5 === dim && address.domain) { - addr += `@${address.domain.toLowerCase()}`; - } - - return addr; -} - function getMessageSerialNumber(message) { return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + message.messageId)).toString(16)).substr(-8); @@ -235,7 +173,8 @@ function getMessageSerialNumber(message) { // ENiGMA½: .@<5dFtnAddress> // function getMessageIdentifier(message, address) { - return `${message.messageId}.${message.areaTag.toLowerCase()}@${formatAddress(address, '5D')} ${getMessageSerialNumber(message)}`; + const addrStr = new Address(address).toString('5D'); + return `${message.messageId}.${message.areaTag.toLowerCase()}@${addrStr} ${getMessageSerialNumber(message)}`; } // @@ -288,5 +227,113 @@ function getOrigin(address) { Config.messageNetworks.originName : Config.general.boardName; - return ` * Origin: ${origin} (${formatAddress(address, '5D')})`; + const addrStr = new Address(address).toString('5D'); + return ` * Origin: ${origin} (${addrStr})`; +} + +function getTearLine() { + return `--- ENiGMA 1/2 v{$packageJson.version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; +} + +function getAbbreviatedNetNodeList(netNodes) { + let abbrList = ''; + let currNet; + netNodes.forEach(netNode => { + if(currNet !== netNode.net) { + abbrList += `${netNode.net}/`; + currNet = netNode.net; + } + abbrList += `${netNode.node} `; + }); + + return abbrList.trim(); // remove trailing space +} + +function parseAbbreviatedNetNodeList(netNodes) { + // + // Make sure we have an array of objects. + // Allow for a single object or string(s) + // + if(!_.isArray(netNodes)) { + if(_.isString(netNodes)) { + netNodes = netNodes.split(' '); + } else { + netNodes = [ netNodes ]; + } + } + + // + // Convert any strings to parsed address objects + // + return netNodes.map(a => { + if(_.isObject(a)) { + return a; + } else { + return Address.fromString(a); + } + }); +} + +// +// Return a FTS-0004.001 SEEN-BY entry(s) that include +// all pre-existing SEEN-BY entries with the addition +// of |additions|. +// +// See http://ftsc.org/docs/fts-0004.001 +// and notes at http://ftsc.org/docs/fsc-0043.002. +// +// For a great write up, see http://www.skepticfiles.org/aj/basics03.htm +// +// This method returns an sorted array of values, but +// not the "SEEN-BY" prefix itself +// +function getUpdatedSeenByEntries(existingEntries, additions) { + /* + From FTS-0004: + + "There can be many seen-by lines at the end of Conference + Mail messages, and they are the real "meat" of the control + information. They are used to determine the systems to + receive the exported messages. The format of the line is: + + SEEN-BY: 132/101 113 136/601 1014/1 + + The net/node numbers correspond to the net/node numbers of + the systems having already received the message. In this way + a message is never sent to a system twice. In a conference + with many participants the number of seen-by lines can be + very large. This line is added if it is not already a part + of the message, or added to if it already exists, each time + a message is exported to other systems. This is a REQUIRED + field, and Conference Mail will not function correctly if + this field is not put in place by other Echomail compatible + programs." + */ + existingEntries = existingEntries || []; + if(!_.isArray(existingEntries)) { + existingEntries = [ existingEntries ]; + } + + additions = parseAbbreviatedNetNodeList(additions).sort(Address.getComparator()); + + // + // For now, we'll just append a new SEEN-BY entry + // + // :TODO: we should at least try and update what is already there in a smart way + existingEntries.push(getAbbreviatedNetNodeList(additions)); + return existingEntries; +} + +function getUpdatedPathEntries(existingEntries, localAddress) { + // :TODO: append to PATH in a smart way! We shoudl try to fit at least the last existing line + + existingEntries = existingEntries || []; + if(!_.isArray(existingEntries)) { + existingEntries = [ existingEntries ]; + } + + existingEntries.push(getAbbreviatedNetNodeList( + parseAbbreviatedNetNodeList(localAddress))); + + return existingEntries; } diff --git a/core/message.js b/core/message.js index 4e0bd0a9..f506f91b 100644 --- a/core/message.js +++ b/core/message.js @@ -109,8 +109,9 @@ Message.FtnPropertyNames = { FtnDestNode : 'ftn_dest_node', FtnOrigNetwork : 'ftn_orig_network', FtnDestNetwork : 'ftn_dest_network', - FtnAttrFlags1 : 'ftn_attr_flags1', - FtnAttrFlags2 : 'ftn_attr_flags2', + FtnAttrFlags : 'ftn_attr_flags', + //FtnAttrFlags1 : 'ftn_attr_flags1', + //FtnAttrFlags2 : 'ftn_attr_flags2', FtnCost : 'ftn_cost', FtnOrigZone : 'ftn_orig_zone', FtnDestZone : 'ftn_dest_zone', diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 5b2283e5..57b6f94e 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -2,8 +2,12 @@ 'use strict'; // ENiGMA½ -var MessageScanTossModule = require('../scan_toss_module.js').MessageScanTossModule; -var Config = require('../config.js').config; +let MessageScanTossModule = require('../scan_toss_module.js').MessageScanTossModule; +let Config = require('../config.js').config; +let ftnMailpacket = require('../ftn_mail_packet.js'); +let ftnUtil = require('../ftn_util.js'); + +let moment = require('moment'); exports.moduleInfo = { name : 'FTN', @@ -18,7 +22,60 @@ function FTNMessageScanTossModule() { this.config = Config.scannerTossers.ftn_bso; - + this.createMessagePacket = function(message, config) { + this.prepareMessage(message); + + let packet = new ftnMailPacket.Packet(); + + let packetHeader = new ftnMailpacket.PacketHeader(); + packetHeader.init( + config.network.localAddress, + config.remoteAddress); + + packetHeader.setPassword(config.remoteNode.packetPassword || ''); + }; + + this.prepareMessage = function(message, config) { + // + // Set various FTN kludges/etc. + // + message.meta.FtnProperty = message.meta.FtnProperty || {}; + message.meta.FtnProperty.ftn_orig_node = config.network.localAddress.node; + message.meta.FtnProperty.ftn_dest_node = config.remoteAddress.node; + message.meta.FtnProperty.ftn_orig_network = config.network.localAddress.net; + message.meta.FtnProperty.ftn_dest_network = config.remoteAddress.net; + // :TODO: attr1 & 2 + message.meta.FtnProperty.ftn_cost = 0; + + message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); + message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(config.network.localAddress); + + if(message.areaTag) { + message.meta.FtnProperty.ftn_area = message.areaTag; + } else { + // :TODO: add "Via" line -- FSP-1010 + } + + // + // When exporting messages, we should create/update SEEN-BY + // with remote address(s) we are exporting to. + // + message.meta.FtnProperty.ftn_seen_by = + ftnUtil.getUpdatedSeenByEntries( + message.meta.FtnProperty.ftn_seen_by, + Config.messageNetworks.ftn.areas[message.areaTag].uplinks + ); + + // + // And create/update PATH for ourself + // + message.meta.FtnKludge['PATH'] = + ftnUtil.getUpdatedPathEntries( + message.meta.FtnKludge['PATH'], + config.network.localAddress.node + ); + }; + } require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); diff --git a/core/string_util.js b/core/string_util.js index f98d0d5a..02c28435 100644 --- a/core/string_util.js +++ b/core/string_util.js @@ -1,14 +1,16 @@ /* jslint node: true */ 'use strict'; -var miscUtil = require('./misc_util.js'); +let miscUtil = require('./misc_util.js'); +let iconv = require('iconv-lite'); -exports.stylizeString = stylizeString; -exports.pad = pad; -exports.replaceAt = replaceAt; -exports.isPrintable = isPrintable; -exports.debugEscapedString = debugEscapedString; +exports.stylizeString = stylizeString; +exports.pad = pad; +exports.replaceAt = replaceAt; +exports.isPrintable = isPrintable; +exports.debugEscapedString = debugEscapedString; +exports.stringFromNullTermBuffer = stringFromNullTermBuffer; // :TODO: create Unicode verison of this var VOWELS = [ 'a', 'e', 'i', 'o', 'u' ]; @@ -176,6 +178,23 @@ function debugEscapedString(s) { return JSON.stringify(s).slice(1, -1); } +function stringFromNullTermBuffer(buf, encoding) { + /*var nullPos = buf.length; + for(var i = 0; i < buf.length; ++i) { + if(0x00 === buf[i]) { + nullPos = i; + break; + } + } + */ + let nullPos = buf.indexOf(new Buffer( [ 0x00 ] )); + if(-1 === nullPos) { + nullPos = buf.length; + } + + return iconv.decode(buf.slice(0, nullPos), encoding || 'utf-8'); +} + // // Extend String.format's object syntax with some modifiers // e.g.: '{username!styleL33t}'.format( { username : 'Leet User' } ) -> "L33t U53r" From 74f5342997d65f08d472d7bb6604c04805e0e041 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 16 Feb 2016 22:11:55 -0700 Subject: [PATCH 05/27] * msg_network.js: Management of message network modules (start/stop/etc.) * Minor updates to ES6 in some areas * Better bbs.js startup seq * Better iterator support for loadModulesForCategory() * Start work on loading message network modules & tieing in record() (WIP) * FTN PacketHeader is now a ES6 class * Various FTN utils, e.g. Via line creation --- core/bbs.js | 77 +++++++++--------- core/config.js | 17 ++-- core/fse.js | 2 +- core/ftn_address.js | 31 ++++++- core/ftn_mail_packet.js | 138 +++++++++++++++++++++++++++----- core/ftn_util.js | 26 ++++++ core/message_area.js | 31 +++++-- core/module_util.js | 43 ++++++---- core/msg_network.js | 60 ++++++++++++++ core/scanner_tossers/ftn_bso.js | 59 +++++++++++--- 10 files changed, 388 insertions(+), 96 deletions(-) create mode 100644 core/msg_network.js diff --git a/core/bbs.js b/core/bbs.js index 5c61e014..efc04cac 100644 --- a/core/bbs.js +++ b/core/bbs.js @@ -5,36 +5,37 @@ //SegfaultHandler.registerHandler('enigma-bbs-segfault.log'); // ENiGMA½ -var conf = require('./config.js'); -var logger = require('./logger.js'); -var miscUtil = require('./misc_util.js'); -var database = require('./database.js'); -var clientConns = require('./client_connections.js'); +let conf = require('./config.js'); +let logger = require('./logger.js'); +let miscUtil = require('./misc_util.js'); +let database = require('./database.js'); +let clientConns = require('./client_connections.js'); -var paths = require('path'); -var async = require('async'); -var util = require('util'); -var _ = require('lodash'); -var assert = require('assert'); -var mkdirp = require('mkdirp'); +let paths = require('path'); +let async = require('async'); +let util = require('util'); +let _ = require('lodash'); +let assert = require('assert'); +let mkdirp = require('mkdirp'); -exports.bbsMain = bbsMain; +// our main entry point +exports.bbsMain = bbsMain; function bbsMain() { async.waterfall( [ function processArgs(callback) { - const args = parseArgs(); + const args = process.argv.slice(2); var configPath; if(args.indexOf('--help') > 0) { // :TODO: display help } else { - var argCount = args.length; - for(var i = 0; i < argCount; ++i) { - var arg = args[i]; - if('--config' == arg) { + let argCount = args.length; + for(let i = 0; i < argCount; ++i) { + const arg = args[i]; + if('--config' === arg) { configPath = args[i + 1]; } } @@ -70,25 +71,19 @@ function bbsMain() { } callback(err); }); + }, + function listenConnections(callback) { + startListening(callback); } ], function complete(err) { - if(!err) { - startListening(); + if(err) { + logger.log.error(err); } } ); } -function parseArgs() { - var args = []; - process.argv.slice(2).forEach(function(val, index, array) { - args.push(val); - }); - - return args; -} - function initialize(cb) { async.series( [ @@ -171,6 +166,9 @@ function initialize(cb) { }); } }); + }, + function readyMessageNetworkSupport(callback) { + require('./msg_network.js').startup(callback); } ], function onComplete(err) { @@ -179,29 +177,30 @@ function initialize(cb) { ); } -function startListening() { +function startListening(cb) { if(!conf.config.servers) { // :TODO: Log error ... output to stderr as well. We can do it all with the logger - logger.log.error('No servers configured'); - return []; + //logger.log.error('No servers configured'); + cb(new Error('No servers configured')); + return; } - var moduleUtil = require('./module_util.js'); // late load so we get Config + let moduleUtil = require('./module_util.js'); // late load so we get Config - moduleUtil.loadModulesForCategory('servers', function onServerModule(err, module) { + moduleUtil.loadModulesForCategory('servers', (err, module) => { if(err) { logger.log.info(err); return; } - var port = parseInt(module.runtime.config.port); + const port = parseInt(module.runtime.config.port); if(isNaN(port)) { logger.log.error( { port : module.runtime.config.port, server : module.moduleInfo.name }, 'Cannot load server (Invalid port)'); return; } - var moduleInst = new module.getModule(); - var server = moduleInst.createServer(); + const moduleInst = new module.getModule(); + const server = moduleInst.createServer(); // :TODO: handle maxConnections, e.g. conf.maxConnections @@ -262,7 +261,11 @@ function startListening() { }); server.listen(port); - logger.log.info({ server : module.moduleInfo.name, port : port }, 'Listening for connections'); + + logger.log.info( + { server : module.moduleInfo.name, port : port }, 'Listening for connections'); + }, err => { + cb(err); }); } diff --git a/core/config.js b/core/config.js index 61de4244..468732c9 100644 --- a/core/config.js +++ b/core/config.js @@ -228,12 +228,6 @@ function getDefaultConfig() { } }, - messages : { - areas : [ - { name : 'private_mail', desc : 'Private Email', groups : [ 'users' ] } - ] - }, - networks : { /* networkName : { // e.g. fidoNet @@ -247,6 +241,17 @@ function getDefaultConfig() { } */ }, + + scannerTossers : { + ftn_bso : { + paths : { + + }, + + maxPacketByteSize : 256000, + maxBundleByteSize : 256000, + } + }, misc : { idleLogoutSeconds : 60 * 6, // 6m diff --git a/core/fse.js b/core/fse.js index 317485e9..f6aa7e7c 100644 --- a/core/fse.js +++ b/core/fse.js @@ -309,7 +309,7 @@ function FullScreenEditorModule(options) { // in NetRunner: self.client.term.rawWrite(ansi.reset() + ansi.deleteLine(3)); - //self.client.term.rawWrite(ansi.reset() + ansi.eraseLine(2)) + self.client.term.rawWrite(ansi.reset() + ansi.eraseLine(2)) } callback(null); }, diff --git a/core/ftn_address.js b/core/ftn_address.js index e6dad9e9..880828d9 100644 --- a/core/ftn_address.js +++ b/core/ftn_address.js @@ -34,7 +34,7 @@ module.exports = class Address { ); } - isMatch(pattern) { + getMatchAddr(pattern) { const m = FTN_PATTERN_REGEXP.exec(pattern); if(m) { let addr = { }; @@ -81,6 +81,35 @@ module.exports = class Address { addr.domain = '*'; } + return addr; + } + } + + /* + getMatchScore(pattern) { + let score = 0; + const addr = this.getMatchAddr(pattern); + if(addr) { + const PARTS = [ 'net', 'node', 'zone', 'point', 'domain' ]; + for(let i = 0; i < PARTS.length; ++i) { + const member = PARTS[i]; + if(this[member] === addr[member]) { + score += 2; + } else if('*' === addr[member]) { + score += 1; + } else { + break; + } + } + } + + return score; + } + */ + + isMatch(pattern) { + const addr = this.getMatchAddr(pattern); + if(addr) { return ( ('*' === addr.net || this.net === addr.net) && ('*' === addr.node || this.node === addr.node) && diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 51215c5d..74950582 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -1,7 +1,6 @@ /* jslint node: true */ 'use strict'; -//var MailPacket = require('./mail_packet.js'); let ftn = require('./ftn_util.js'); let Message = require('./message.js'); let sauce = require('./sauce.js'); @@ -18,7 +17,6 @@ let iconv = require('iconv-lite'); let buffers = require('buffers'); let moment = require('moment'); -exports.PacketHeader = PacketHeader; exports.Packet = Packet; /* @@ -40,6 +38,117 @@ const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; +class PacketHeader { + constructor(origAddr, destAddr, created, version) { + const EMPTY_ADDRESS = { + node : 0, + net : 0, + zone : 0, + point : 0, + }; + + this.packetVersion = version || '2+'; + + this.origAddress = origAddr || EMPTY_ADDRESS; + this.destAddress = destAddr || EMPTY_ADDRESS; + this.created = created || moment(); + + // uncommon to set the following explicitly + this.prodCodeLo = 0xfe; // http://ftsc.org/docs/fta-1005.003 + this.prodRevLo = 0; + this.baud = 0; + this.packetType = FTN_PACKET_HEADER_TYPE; + this.password = ''; + this.prodData = 0x47694e45; // "ENiG" + + this.capWord = 0x0001; + this.capWordValidate = ((this.capWord & 0xff) << 8) | ((this.capWord >> 8) & 0xff); + } + + get origAddress() { + let addr = new Address({ + node : this.origNode, + zone : this.origZone, + }); + + if(this.origPoint) { + addr.point = this.origPoint; + addr.net = this.auxNet; + } else { + addr.net = this.origNet; + } + + return addr; + } + + set origAddress(address) { + if(_.isString(address)) { + address = Address.fromString(address); + } + + this.origNode = address.node; + + // See FSC-48 + if(address.point) { + this.origNet = -1; + this.auxNet = address.net; + } else { + this.origNet = address.net; + this.auxNet = 0; + } + + this.origZone = address.zone; + this.origZone2 = address.zone; + this.origPoint = address.point || 0; + } + + get destAddress() { + let addr = new Address({ + node : this.destNode, + net : this.destNet, + zone : this.destZone, + }); + + if(this.destPoint) { + addr.point = this.destPoint; + } + + return addr; + } + + set destAddress(address) { + if(_.isString(address)) { + address = Address.fromString(address); + } + + this.destNode = address.node; + this.destNet = address.net; + this.destZone = address.zone; + this.destZone2 = address.zone; + this.destPoint = address.point || 0; + } + + get created() { + return moment(this); // use year, month, etc. properties + } + + set created(momentCreated) { + if(!moment.isMoment(momentCreated)) { + created = moment(momentCreated); + } + + this.year = momentCreated.year(); + this.month = momentCreated.month(); + this.day = momentCreated.day(); + this.hour = momentCreated.hour(); + this.minute = momentCreated.minute(); + this.second = momentCreated.second(); + } +} + +exports.PacketHeader = PacketHeader; + +/* function PacketHeader(options) { } @@ -120,7 +229,7 @@ PacketHeader.prototype.setCreated = function(created) { PacketHeader.prototype.setPassword = function(password) { this.password = password.substr(0, 8); } - +*/ // // Read/Write FTN packets with support for the following formats: @@ -762,7 +871,7 @@ Packet.prototype.write = function(path, packetHeader, messages) { ); }; - +/* const LOCAL_ADDRESS = { zone : 46, net : 1, @@ -776,9 +885,8 @@ const REMOTE_ADDRESS = { node : 218, }; -var packetHeader1 = new PacketHeader(); -packetHeader1.init(LOCAL_ADDRESS, REMOTE_ADDRESS); -console.log(packetHeader1); + +var packetHeader1 = new PacketHeader(LOCAL_ADDRESS, REMOTE_ADDRESS); var packet = new Packet(); var theHeader; @@ -796,23 +904,13 @@ packet.read( messagesToWrite.push(msg); - /* - if(!written) { - written = true; - - let messages = [ msg ]; - Packet.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messages, err => { - - }); - - }*/ - let address = { zone : 46, net : 1, node : 232, domain : 'l33t.codes', }; + msg.areaTag = 'agn_bbs'; msg.messageId = 1234; console.log(ftn.getMessageIdentifier(msg, address)); @@ -825,6 +923,9 @@ packet.read( msg.meta.FtnProperty.ftn_seen_by, '1/107 4/22 4/25 4/10')); console.log(ftn.getUpdatedPathEntries( msg.meta.FtnKludge['PATH'], '1:365/50')) + console.log('Via: ' + ftn.getVia(address)) + console.log(Address.fromString('46:1/232.4@l33t.codes').isMatch('*:1/232.*')) + //console.log(Address.fromString('46:1/232.4@l33t.codes').getMatchScore('46:1/232')) } }, function completion(err) { @@ -837,3 +938,4 @@ packet.read( }); } ); +*/ \ No newline at end of file diff --git a/core/ftn_util.js b/core/ftn_util.js index a5cec2a2..dbc1d2ea 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -28,6 +28,7 @@ exports.getProductIdentifier = getProductIdentifier; exports.getUTCTimeZoneOffset = getUTCTimeZoneOffset; exports.getOrigin = getOrigin; exports.getTearLine = getTearLine; +exports.getVia = getVia; exports.getAbbreviatedNetNodeList = getAbbreviatedNetNodeList; exports.parseAbbreviatedNetNodeList = parseAbbreviatedNetNodeList; exports.getUpdatedSeenByEntries = getUpdatedSeenByEntries; @@ -181,6 +182,9 @@ function getMessageIdentifier(message, address) { // Return a FSC-0046.005 Product Identifier or "PID" // http://ftsc.org/docs/fsc-0046.005 // +// Note that we use a variant on the spec for +// in which (; ; ) is used instead +// function getProductIdentifier() { const version = packageJson.version .replace(/\-/g, '.') @@ -235,6 +239,28 @@ function getTearLine() { return `--- ENiGMA 1/2 v{$packageJson.version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; } +// +// Return a FRL-1005.001 "Via" line +// http://ftsc.org/docs/frl-1005.001 +// +function getVia(address) { + /* + FRL-1005.001 states teh following format: + + ^AVia: @YYYYMMDD.HHMMSS[.Precise][.Time Zone] + [Serial Number] + */ + const addrStr = new Address(address).toString('5D'); + const dateTime = moment().utc().format('YYYYMMDD.HHmmSS.SSSS.UTC'); + + const version = packageJson.version + .replace(/\-/g, '.') + .replace(/alpha/,'a') + .replace(/beta/,'b'); + + return `${addrStr} @${dateTime} ENiGMA1/2 ${version}`; +} + function getAbbreviatedNetNodeList(netNodes) { let abbrList = ''; let currNet; diff --git a/core/message_area.js b/core/message_area.js index 957a0e37..06bb5234 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -1,15 +1,16 @@ /* jslint node: true */ 'use strict'; -var msgDb = require('./database.js').dbs.message; -var Config = require('./config.js').config; -var Message = require('./message.js'); -var Log = require('./logger.js').log; -var checkAcs = require('./acs_util.js').checkAcs; +let msgDb = require('./database.js').dbs.message; +let Config = require('./config.js').config; +let Message = require('./message.js'); +let Log = require('./logger.js').log; +let checkAcs = require('./acs_util.js').checkAcs; +let msgNetRecord = require('./msg_network.js').recordMessage; -var async = require('async'); -var _ = require('lodash'); -var assert = require('assert'); +let async = require('async'); +let _ = require('lodash'); +let assert = require('assert'); exports.getAvailableMessageConferences = getAvailableMessageConferences; exports.getSortedAvailMessageConferences = getSortedAvailMessageConferences; @@ -439,3 +440,17 @@ function updateMessageAreaLastReadId(userId, areaTag, messageId, cb) { } ); } + +function persistMessage(message, cb) { + async.series( + [ + function persistMessageToDisc(callback) { + message.persist(callback); + }, + function recordToMessageNetworks(callback) { + msgNetRecord(message, callback); + } + ], + cb + ); +} \ No newline at end of file diff --git a/core/module_util.js b/core/module_util.js index c66155f1..fa28d634 100644 --- a/core/module_util.js +++ b/core/module_util.js @@ -1,20 +1,22 @@ /* jslint node: true */ 'use strict'; -var Config = require('./config.js').config; -var miscUtil = require('./misc_util.js'); +// ENiGMA½ +let Config = require('./config.js').config; +let miscUtil = require('./misc_util.js'); -var fs = require('fs'); -var paths = require('path'); -var _ = require('lodash'); -var assert = require('assert'); +// standard/deps +let fs = require('fs'); +let paths = require('path'); +let _ = require('lodash'); +let assert = require('assert'); +let async = require('async'); // exports exports.loadModuleEx = loadModuleEx; exports.loadModule = loadModule; exports.loadModulesForCategory = loadModulesForCategory; - function loadModuleEx(options, cb) { assert(_.isObject(options)); assert(_.isString(options.name)); @@ -44,7 +46,7 @@ function loadModuleEx(options, cb) { return; } - // Ref configuration, if any, for convience to the module + // Ref configuration, if any, for convience to the module mod.runtime = { config : modConfig }; cb(null, mod); @@ -63,18 +65,27 @@ function loadModule(name, category, cb) { }); } -function loadModulesForCategory(category, iterator) { - var path = Config.paths[category]; - - fs.readdir(path, function onFiles(err, files) { +function loadModulesForCategory(category, iterator, complete) { + + fs.readdir(Config.paths[category], (err, files) => { if(err) { - cb(err); + iterator(err); return; } - var filtered = files.filter(function onFilter(file) { return '.js' === paths.extname(file); }); - filtered.forEach(function onFile(file) { - loadModule(paths.basename(file, '.js'), category, iterator); + const jsModules = files.filter(file => { + return '.js' === paths.extname(file); + }); + + async.each(jsModules, (file, next) => { + loadModule(paths.basename(file, '.js'), category, (err, mod) => { + iterator(err, mod); + next(); + }); + }, err => { + if(complete) { + complete(err); + } }); }); } diff --git a/core/msg_network.js b/core/msg_network.js new file mode 100644 index 00000000..5354ffdc --- /dev/null +++ b/core/msg_network.js @@ -0,0 +1,60 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +let loadModulesForCategory = require('./module_util.js').loadModulesForCategory; + +// standard/deps +let async = require('async'); + +exports.startup = startup +exports.shutdown = shutdown; +exports.recordMessage = recordMessage; + +let msgNetworkModules = []; + +function startup(cb) { + async.series( + [ + function loadModules(callback) { + loadModulesForCategory('scannerTossers', (err, module) => { + if(!err) { + const modInst = new module.getModule(); + + modInst.startup(err => { + if(!err) { + msgNetworkModules.push(modInst); + } + }); + } + }, err => { + callback(err); + }); + } + ], + cb + ); +} + +function shutdown() { + msgNetworkModules.forEach(mod => { + mod.shutdown(); + }); + + msgNetworkModules = []; +} + +function recordMessage(message, cb) { + // + // Give all message network modules (scanner/tossers) + // a chance to do something with |message|. Any or all can + // choose to ignore it. + // + async.each(msgNetworkModules, (modInst, next) => { + modInst.record(message, err => { + next(); + }); + }, err => { + cb(err); + }); +} \ No newline at end of file diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 57b6f94e..3cc6c0cf 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -2,12 +2,15 @@ 'use strict'; // ENiGMA½ -let MessageScanTossModule = require('../scan_toss_module.js').MessageScanTossModule; -let Config = require('../config.js').config; -let ftnMailpacket = require('../ftn_mail_packet.js'); -let ftnUtil = require('../ftn_util.js'); +let MessageScanTossModule = require('../msg_scan_toss_module.js').MessageScanTossModule; +let Config = require('../config.js').config; +let ftnMailpacket = require('../ftn_mail_packet.js'); +let ftnUtil = require('../ftn_util.js'); +let Address = require('../ftn_address.js'); +let Log = require('../logger.js').log; -let moment = require('moment'); +let moment = require('moment'); +let _ = require('lodash'); exports.moduleInfo = { name : 'FTN', @@ -20,7 +23,9 @@ exports.getModule = FTNMessageScanTossModule; function FTNMessageScanTossModule() { MessageScanTossModule.call(this); - this.config = Config.scannerTossers.ftn_bso; + if(_.has(Config, 'scannerTossers.ftn_bso')) { + this.config = Config.scannerTossers.ftn_bso; + } this.createMessagePacket = function(message, config) { this.prepareMessage(message); @@ -50,10 +55,18 @@ function FTNMessageScanTossModule() { message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(config.network.localAddress); - if(message.areaTag) { - message.meta.FtnProperty.ftn_area = message.areaTag; + if(message.isPrivate()) { + // + // NetMail messages need a FRL-1005.001 "Via" line + // http://ftsc.org/docs/frl-1005.001 + // + if(_.isString(message.meta.FtnKludge['Via'])) { + message.meta.FtnKludge['Via'] = [ message.meta.FtnKludge['Via'] ]; + } + message.meta.FtnKludge['Via'] = message.meta.FtnKludge['Via'] || []; + message.meta.FtnKludge['Via'].push(ftnUtil.getVia(config.network.localAddress)); } else { - // :TODO: add "Via" line -- FSP-1010 + message.meta.FtnProperty.ftn_area = message.areaTag; } // @@ -81,14 +94,42 @@ function FTNMessageScanTossModule() { require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); FTNMessageScanTossModule.prototype.startup = function(cb) { + Log.info('FidoNet Scanner/Tosser starting up'); + cb(null); }; FTNMessageScanTossModule.prototype.shutdown = function(cb) { + Log.info('FidoNet Scanner/Tosser shutting down'); + cb(null); }; FTNMessageScanTossModule.prototype.record = function(message, cb) { + if(!_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) { + return; + } + + const area = Config.messageNetworks.ftn.areas[message.areaTag]; + if(!_.isString(area.ftnArea) || !_.isArray(area.uplinks)) { + // :TODO: should probably log a warning here + return; + } + + // + // For each uplink, find the best configuration match + // + area.uplinks.forEach(uplink => { + // :TODO: sort by least # of '*' & take top? + let matchNodes = _.filter(Object.keys(Config.scannerTossers.ftn_bso.nodes), addr => { + return Address.fromString(addr).isMatch(uplink); + }); + + if(matchNodes.length > 0) { + const nodeKey = matchNodes[0]; + + } + }); cb(null); From 75698f62afc7c4c225e7a32ddfcb59cd34649e8c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 17 Feb 2016 20:44:46 -0700 Subject: [PATCH 06/27] Add ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..a34168f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,16 @@ +For :bug: bug reports, please fill out the information below plus any additional relevant information. If this is a feature request, feel free to clear the form. + +**Short problem description** + +**Environment** +- [ ] I am using Node.js v4.x or higher +- [ ] `npm install` reports success +- Actual Node.js version (`node --version`): +- Operating system (`uname -a` on *nix systems): +- Revision (`git rev-parse --short HEAD`): + +**Expected behavior** + +**Actual behavior** + +**Steps to reproduce** From a858a93ee197f7049eb168ea1760d6e82c5fcae1 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 20 Feb 2016 17:57:38 -0700 Subject: [PATCH 07/27] * FTN BSO module: Export to . dirs where appropriate * Code cleanup * Fix FTN packet header writing * Add CHRS support to FTN packet I/O * Change to FNV-1a hash of ms since 2016-1-1 ("enigma epoc") + message ID for MSGID serial number and .pkt BSO export * Only write some FTN kludges for EchoMail (vs NetMail) * If config specifies, call message network modoule(s) .record() method @ persist (WIP) --- core/config.js | 18 +- core/fnv1a.js | 50 ++++++ core/ftn_mail_packet.js | 290 +++++++------------------------- core/ftn_util.js | 115 +++++++++++-- core/message_area.js | 1 + core/msg_network.js | 5 +- core/msg_scan_toss_module.js | 3 +- core/scanner_tossers/ftn_bso.js | 239 ++++++++++++++++++++------ mods/msg_area_post_fse.js | 14 +- 9 files changed, 413 insertions(+), 322 deletions(-) create mode 100644 core/fnv1a.js diff --git a/core/config.js b/core/config.js index 468732c9..85865ee3 100644 --- a/core/config.js +++ b/core/config.js @@ -227,25 +227,13 @@ function getDefaultConfig() { } } }, - - networks : { - /* - networkName : { // e.g. fidoNet - address : { - zone : 0, - net : 0, - node : 0, - point : 0, - domain : 'l33t.codes' - } - } - */ - }, scannerTossers : { ftn_bso : { paths : { - + outbound : paths.join(__dirname, './../mail/out/'), + inbound : paths.join(__dirname, './../mail/in/'), + secInbound : paths.join(__dirname, './../mail/secin/'), }, maxPacketByteSize : 256000, diff --git a/core/fnv1a.js b/core/fnv1a.js new file mode 100644 index 00000000..f7714936 --- /dev/null +++ b/core/fnv1a.js @@ -0,0 +1,50 @@ +/* jslint node: true */ +'use strict'; + +let _ = require('lodash'); + +// FNV-1a based on work here: https://github.com/wiedi/node-fnv +module.exports = class FNV1a { + constructor(data) { + this.hash = 0x811c9dc5; + + if(!_.isUndefined(data)) { + this.update(data); + } + } + + update(data) { + if(_.isNumber(data)) { + data = data.toString(); + } + + if(_.isString(data)) { + data = new Buffer(data); + } + + if(!Buffer.isBuffer(data)) { + throw new Error('data must be String or Buffer!'); + } + + for(let b of data) { + this.hash = this.hash ^ b; + this.hash += + (this.hash << 24) + (this.hash << 8) + (this.hash << 7) + + (this.hash << 4) + (this.hash << 1); + } + + return this; + } + + digest(encoding) { + encoding = encoding || 'binary'; + let buf = new Buffer(4); + buf.writeInt32BE(this.hash & 0xffffffff, 0); + return buf.toString(encoding); + } + + get value() { + return this.hash & 0xffffffff; + } +} + diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 74950582..34801e6c 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -39,7 +39,7 @@ const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; class PacketHeader { - constructor(origAddr, destAddr, created, version) { + constructor(origAddr, destAddr, version, created) { const EMPTY_ADDRESS = { node : 0, net : 0, @@ -139,7 +139,7 @@ class PacketHeader { this.year = momentCreated.year(); this.month = momentCreated.month(); - this.day = momentCreated.day(); + this.day = momentCreated.date(); // day of month this.hour = momentCreated.hour(); this.minute = momentCreated.minute(); this.second = momentCreated.second(); @@ -148,89 +148,6 @@ class PacketHeader { exports.PacketHeader = PacketHeader; -/* -function PacketHeader(options) { -} - -PacketHeader.prototype.init = function(origAddr, destAddr, created, version) { - version = version || '2+'; - - const EMPTY_ADDRESS = { - node : 0, - net : 0, - zone : 0, - point : 0, - }; - - this.packetVersion = version; - - this.setOrigAddress(origAddr || EMPTY_ADDRESS); - this.setDestAddress(destAddr || EMPTY_ADDRESS); - this.setCreated(created || moment()); - - // uncommon to set the following explicitly - this.prodCodeLo = 0xfe; // http://ftsc.org/docs/fta-1005.003 - this.prodRevLo = 0; - this.baud = 0; - this.packetType = FTN_PACKET_HEADER_TYPE; - this.password = ''; - this.prodData = 0x47694e45; // "ENiG" - - this.capWord = 0x0001; - this.capWordValidate = ((this.capWord & 0xff) << 8) | ((this.capWord >> 8) & 0xff); - - return this; -}; - -PacketHeader.prototype.setOrigAddress = function(address) { - this.origNode = address.node; - - // See FSC-48 - if(address.point) { - this.origNet = -1; - this.auxNet = address.net; - } else { - this.origNet = address.net; - this.auxNet = 0; - } - - this.origZone = address.zone; - this.origZone2 = address.zone; - this.origPoint = address.point || 0; - - return this; -}; - -PacketHeader.prototype.setDestAddress = function(address) { - this.destNode = address.node; - this.destNet = address.net; - this.destZone = address.zone; - this.destZone2 = address.zone; - this.destPoint = address.point || 0; - - return this; -}; - -PacketHeader.prototype.setCreated = function(created) { - if(!moment.isMoment(created)) { - created = moment(created); - } - - this.year = created.year(); - this.month = created.month(); - this.day = created.day(); - this.hour = created.hour(); - this.minute = created.minute(); - this.second = created.second(); - - return this; -}; - -PacketHeader.prototype.setPassword = function(password) { - this.password = password.substr(0, 8); -} -*/ - // // Read/Write FTN packets with support for the following formats: // @@ -338,15 +255,19 @@ function Packet() { } } - // - // Date/time components into something more reasonable - // Note: The names above match up with object members moment() allows - // - packetHeader.created = moment(packetHeader); + packetHeader.created = moment({ + year : packetHeader.year, + month : packetHeader.month, + date : packetHeader.day, + hour : packetHeader.hour, + minute : packetHeader.minute, + second : packetHeader.second + }); let ph = new PacketHeader(); _.assign(ph, packetHeader); + cb(null, ph); }); }; @@ -358,7 +279,7 @@ function Packet() { buffer.writeUInt16LE(packetHeader.destNode, 2); buffer.writeUInt16LE(packetHeader.created.year(), 4); buffer.writeUInt16LE(packetHeader.created.month(), 6); - buffer.writeUInt16LE(packetHeader.created.date(), 8); + buffer.writeUInt16LE(packetHeader.created.date(), 8); // day of month buffer.writeUInt16LE(packetHeader.created.hour(), 10); buffer.writeUInt16LE(packetHeader.created.minute(), 12); buffer.writeUInt16LE(packetHeader.created.second(), 14); @@ -374,45 +295,6 @@ function Packet() { buffer.writeUInt16LE(packetHeader.origZone, 34); buffer.writeUInt16LE(packetHeader.destZone, 36); - - // :TODO: update header information appropriately for 2 and 2.2 - switch(packetHeader.packetVersion) { - case '2' : - // filler... - packetHeader.auxNet = 0; - packetHeader.capWordValidate = 0; - packetHeader.prodCodeHi = 0; - packetHeader.prodRevLo = 0; - packetHeader.capWord = 0; - packetHeader.origZone2 = 0; - packetHeader.destZone2 = 0; - packetHeader.origPoint = 0; - packetHeader.destPoint = 0; - break; - - case '2.2' : - packetHeader.day = 0; - packetHeader.hour = 0; - packetHeader.minute = 0; - packetHeader.second = 0; - - // :TODO: copy over fields from 2+ -> overriden fields here! - - packetHeader.baud = FTN_PACKET_BAUD_TYPE_2_2; - break; - - case '2+' : - const capWordValidateSwapped = - ((packetHeader.capWordValidate & 0xff) << 8) | - ((packetHeader.capWordValidate >> 8) & 0xff); - - packetHeader.capWordValidate = capWordValidateSwapped; - - // :TODO: set header appropriate if point - break; - - } - buffer.writeUInt16LE(packetHeader.auxNet, 38); buffer.writeUInt16LE(packetHeader.capWordValidate, 40); buffer.writeUInt8(packetHeader.prodCodeHi, 42); @@ -422,12 +304,6 @@ function Packet() { buffer.writeUInt16LE(packetHeader.destZone2, 48); buffer.writeUInt16LE(packetHeader.origPoint, 50); buffer.writeUInt16LE(packetHeader.destPoint, 52); - - // Store in "ENiG" in prodData unless we already have something useful - if(0 === packetHeader.prodData) { - packetHeader.prodData = 0x47694e45; - } - buffer.writeUInt32LE(packetHeader.prodData, 54); ws.write(buffer); @@ -479,6 +355,8 @@ function Packet() { messageBodyData.kludgeLines[key] = value; } } + + let encoding = 'cp437'; async.series( [ @@ -503,10 +381,45 @@ function Packet() { callback(null); } }, + function extractChrsAndDetermineEncoding(callback) { + // + // From FTS-5003.001: + // "The CHRS control line is formatted as follows: + // + // ^ACHRS: + // + // Where is a character string of no more than eight (8) + // ASCII characters identifying the character set or character encoding + // scheme used, and level is a positive integer value describing what + // level of CHRS the message is written in." + // + // Also according to the spec, the deprecated "CHARSET" value may be used + // :TODO: Look into CHARSET more - should we bother supporting it? + // :TODO: See encodingFromHeader() for CHRS/CHARSET support @ https://github.com/Mithgol/node-fidonet-jam + const FTN_CHRS_PREFIX = new Buffer( [ 0x01, 0x43, 0x48, 0x52, 0x53, 0x3a, 0x20 ] ); // "\x01CHRS:" + const FTN_CHRS_SUFFIX = new Buffer( [ 0x0d ] ); + binary.parse(messageBodyBuffer) + .scan('prefix', FTN_CHRS_PREFIX) + .scan('content', FTN_CHRS_SUFFIX) + .tap(chrsData => { + if(chrsData.prefix && chrsData.content && chrsData.content.length > 0) { + const chrs = iconv.decode(chrsData.content, 'CP437'); + const chrsEncoding = ftn.getEncodingFromCharacterSetIdentifier(chrs); + if(chrsEncoding) { + encoding = chrsEncoding; + } + callback(null); + } else { + callback(null); + } + }); + }, function extractMessageData(callback) { - const messageLines = - iconv.decode(messageBodyBuffer, 'CP437').replace(/[\xec\n]/g, '').split(/\r/g); - + // + // Decode |messageBodyBuffer| using |encoding| defaulted or detected above + // + // :TODO: Look into \xec thing more - document + const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/[\xec\n]/g, '').split(/\r/g); let preOrigin = true; messageLines.forEach(line => { @@ -566,8 +479,6 @@ function Packet() { .word16lu('ftn_orig_network') .word16lu('ftn_dest_network') .word16lu('ftn_attr_flags') - //.word8('ftn_attr_flags1') - //.word8('ftn_attr_flags2') .word16lu('ftn_cost') .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max @@ -616,8 +527,6 @@ function Packet() { msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; - //msg.meta.FtnProperty.ftn_attr_flags1 = msgData.ftn_attr_flags1; - //msg.meta.FtnProperty.ftn_attr_flags2 = msgData.ftn_attr_flags2; msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; self.processMessageBody(msgData.message, function processed(messageBodyData) { @@ -661,7 +570,7 @@ function Packet() { }); }; - this.writeMessage = function(message, ws) { + this.writeMessage = function(message, ws, options) { let basicHeader = new Buffer(34); basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); @@ -670,8 +579,6 @@ function Packet() { basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); - //basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags1, 10); - //basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags2, 11); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); @@ -731,7 +638,7 @@ function Packet() { } }); - msgBody += message.message; + msgBody += message.message + '\r'; // // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 @@ -756,7 +663,9 @@ function Packet() { appendMeta('\x01PATH', message.meta.FtnKludge['PATH']); - ws.write(iconv.encode(msgBody + '\0', 'CP437')); + // + // :TODO: We should encode based on config and add the proper kludge here! + ws.write(iconv.encode(msgBody + '\0', options.encoding)); }; this.parsePacketBuffer = function(packetBuffer, iterator, cb) { @@ -839,19 +748,13 @@ Packet.prototype.writeStream = function(ws, messages, options) { } if(_.isObject(options.packetHeader)) { - /* - let packetHeader = options.packetHeader; - - // default header - packetHeader.packetVersion = packetHeader.packetVersion || '2+'; - packetHeader.created = packetHeader.created || moment(); - packetHeader.baud = packetHeader.baud || 0; - */ this.writePacketHeader(options.packetHeader, ws); } + + options.encoding = options.encoding || 'utf8'; messages.forEach(msg => { - this.writeMessage(msg, ws); + this.writeMessage(msg, ws, options); }); if(true === options.terminatePacket) { @@ -859,10 +762,12 @@ Packet.prototype.writeStream = function(ws, messages, options) { } } -Packet.prototype.write = function(path, packetHeader, messages) { +Packet.prototype.write = function(path, packetHeader, messages, options) { if(!_.isArray(messages)) { messages = [ messages ]; } + + options = options || { encoding : 'utf8' }; // utf-8 = 'CHRS UTF-8 4' this.writeStream( fs.createWriteStream(path), // :TODO: specify mode/etc. @@ -870,72 +775,3 @@ Packet.prototype.write = function(path, packetHeader, messages) { { packetHeader : packetHeader, terminatePacket : true } ); }; - -/* -const LOCAL_ADDRESS = { - zone : 46, - net : 1, - node : 232, - domain : 'l33t.codes', -}; - -const REMOTE_ADDRESS = { - zone : 1, - net : 2, - node : 218, -}; - - -var packetHeader1 = new PacketHeader(LOCAL_ADDRESS, REMOTE_ADDRESS); - -var packet = new Packet(); -var theHeader; -var written = false; -let messagesToWrite = []; -packet.read( - process.argv[2], - function iterator(dataType, data) { - if('header' === dataType) { - theHeader = data; - console.log(theHeader); - } else if('message' === dataType) { - const msg = data; - console.log(msg); - - messagesToWrite.push(msg); - - let address = { - zone : 46, - net : 1, - node : 232, - domain : 'l33t.codes', - }; - - msg.areaTag = 'agn_bbs'; - msg.messageId = 1234; - console.log(ftn.getMessageIdentifier(msg, address)); - console.log(ftn.getProductIdentifier()) - //console.log(ftn.getOrigin(address)) - //console.log(ftn.parseAddress('46:1/232.4@l33t.codes')) - console.log(Address.fromString('46:1/232.4@l33t.codes')); - console.log(ftn.getUTCTimeZoneOffset()) - console.log(ftn.getUpdatedSeenByEntries( - msg.meta.FtnProperty.ftn_seen_by, '1/107 4/22 4/25 4/10')); - console.log(ftn.getUpdatedPathEntries( - msg.meta.FtnKludge['PATH'], '1:365/50')) - console.log('Via: ' + ftn.getVia(address)) - console.log(Address.fromString('46:1/232.4@l33t.codes').isMatch('*:1/232.*')) - //console.log(Address.fromString('46:1/232.4@l33t.codes').getMatchScore('46:1/232')) - } - }, - function completion(err) { - console.log('complete!') - console.log(err); - - - console.log(messagesToWrite.length) - packet.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messagesToWrite, err => { - }); - } -); -*/ \ No newline at end of file diff --git a/core/ftn_util.js b/core/ftn_util.js index dbc1d2ea..5172b218 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -1,24 +1,26 @@ /* jslint node: true */ 'use strict'; -var Config = require('./config.js').config; -var Address = require('./ftn_address.js'); +let Config = require('./config.js').config; +let Address = require('./ftn_address.js'); +let FNV1a = require('./fnv1a.js'); -var _ = require('lodash'); -var assert = require('assert'); -var binary = require('binary'); -var fs = require('fs'); -var util = require('util'); -var iconv = require('iconv-lite'); -var moment = require('moment'); -var createHash = require('crypto').createHash; -var uuid = require('node-uuid'); -var os = require('os'); +let _ = require('lodash'); +let assert = require('assert'); +let binary = require('binary'); +let fs = require('fs'); +let util = require('util'); +let iconv = require('iconv-lite'); +let moment = require('moment'); +let createHash = require('crypto').createHash; +let uuid = require('node-uuid'); +let os = require('os'); -var packageJson = require('../package.json'); +let packageJson = require('../package.json'); // :TODO: Remove "Ftn" from most of these -- it's implied in the module exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; +exports.getMessageSerialNumber = getMessageSerialNumber; exports.createMessageUuid = createMessageUuid; exports.getDateFromFtnDateTime = getDateFromFtnDateTime; exports.getDateTimeString = getDateTimeString; @@ -34,6 +36,9 @@ exports.parseAbbreviatedNetNodeList = parseAbbreviatedNetNodeList; exports.getUpdatedSeenByEntries = getUpdatedSeenByEntries; exports.getUpdatedPathEntries = getUpdatedPathEntries; +exports.getCharacterSetIdentifierByEncoding = getCharacterSetIdentifierByEncoding; +exports.getEncodingFromCharacterSetIdentifier = getEncodingFromCharacterSetIdentifier; + exports.getQuotePrefix = getQuotePrefix; // @@ -134,8 +139,11 @@ function createMessageUuid(ftnMsgId, ftnArea) { } function getMessageSerialNumber(message) { - return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + - message.messageId)).toString(16)).substr(-8); + const msSinceEnigmaEpoc = (Date.now() - Date.UTC(2016, 1, 1)); + const hash = Math.abs(new FNV1a(msSinceEnigmaEpoc + message.messageId).value).toString(16); + return `00000000${hash}`.substr(-8); + // return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + + // message.messageId)).toString(16)).substr(-8); } // @@ -236,7 +244,8 @@ function getOrigin(address) { } function getTearLine() { - return `--- ENiGMA 1/2 v{$packageJson.version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; + const nodeVer = process.version.substr(1); // remove 'v' prefix + return `--- ENiGMA 1/2 v${packageJson.version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; } // @@ -363,3 +372,77 @@ function getUpdatedPathEntries(existingEntries, localAddress) { return existingEntries; } + +// +// Return FTS-5000.001 "CHRS" value +// http://ftsc.org/docs/fts-5003.001 +// +const ENCODING_TO_FTS_5003_001_CHARS = { + // level 1 - generally should not be used + ascii : [ 'ASCII', 1 ], + 'us-ascii' : [ 'ASCII', 1 ], + + // level 2 - 8 bit, ASCII based + cp437 : [ 'CP437', 2 ], + cp850 : [ 'CP850', 2 ], + + // level 3 - reserved + + // level 4 + utf8 : [ 'UTF-8', 4 ], + 'utf-8' : [ 'UTF-8', 4 ], +}; + + +function getCharacterSetIdentifierByEncoding(encodingName) { + const value = ENCODING_TO_FTS_5003_001_CHARS[encodingName.toLowerCase()]; + return value ? `${value[0]} ${value[1]}` : encodingName.toUpperCase(); +} + +function getEncodingFromCharacterSetIdentifier(chrs) { + const ident = chrs.split(' ')[0].toUpperCase(); + + // :TODO: fill in the rest!!! + return { + // level 1 + 'ASCII' : 'iso-646-1', + 'DUTCH' : 'iso-646', + 'FINNISH' : 'iso-646-10', + 'FRENCH' : 'iso-646', + 'CANADIAN' : 'iso-646', + 'GERMAN' : 'iso-646', + 'ITALIAN' : 'iso-646', + 'NORWEIG' : 'iso-646', + 'PORTU' : 'iso-646', + 'SPANISH' : 'iso-656', + 'SWEDISH' : 'iso-646-10', + 'SWISS' : 'iso-646', + 'UK' : 'iso-646', + 'ISO-10' : 'iso-646-10', + + // level 2 + 'CP437' : 'cp437', + 'CP850' : 'cp850', + 'CP852' : 'cp852', + 'CP866' : 'cp866', + 'CP848' : 'cp848', + 'CP1250' : 'cp1250', + 'CP1251' : 'cp1251', + 'CP1252' : 'cp1252', + 'CP10000' : 'macroman', + 'LATIN-1' : 'iso-8859-1', + 'LATIN-2' : 'iso-8859-2', + 'LATIN-5' : 'iso-8859-9', + 'LATIN-9' : 'iso-8859-15', + + // level 4 + 'UTF-8' : 'utf8', + + // deprecated stuff + 'IBMPC' : 'cp1250', // :TODO: validate + '+7_FIDO' : 'cp866', + '+7' : 'cp866', + 'MAC' : 'macroman', // :TODO: validate + + }[ident]; +} \ No newline at end of file diff --git a/core/message_area.js b/core/message_area.js index 06bb5234..4d47b31e 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -26,6 +26,7 @@ exports.getMessageListForArea = getMessageListForArea; exports.getNewMessagesInAreaForUser = getNewMessagesInAreaForUser; exports.getMessageAreaLastReadId = getMessageAreaLastReadId; exports.updateMessageAreaLastReadId = updateMessageAreaLastReadId; +exports.persistMessage = persistMessage; const CONF_AREA_RW_ACS_DEFAULT = 'GM[users]'; const AREA_MANAGE_ACS_DEFAULT = 'GM[sysops]'; diff --git a/core/msg_network.js b/core/msg_network.js index 5354ffdc..030c544e 100644 --- a/core/msg_network.js +++ b/core/msg_network.js @@ -51,9 +51,8 @@ function recordMessage(message, cb) { // choose to ignore it. // async.each(msgNetworkModules, (modInst, next) => { - modInst.record(message, err => { - next(); - }); + modInst.record(message); + next(); }, err => { cb(err); }); diff --git a/core/msg_scan_toss_module.js b/core/msg_scan_toss_module.js index e396f44d..8172d77f 100644 --- a/core/msg_scan_toss_module.js +++ b/core/msg_scan_toss_module.js @@ -20,6 +20,5 @@ MessageScanTossModule.prototype.shutdown = function(cb) { cb(null); }; -MessageScanTossModule.prototype.record = function(message, cb) { - cb(null); +MessageScanTossModule.prototype.record = function(message) { }; \ No newline at end of file diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 3cc6c0cf..71e95aae 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -4,13 +4,15 @@ // ENiGMA½ let MessageScanTossModule = require('../msg_scan_toss_module.js').MessageScanTossModule; let Config = require('../config.js').config; -let ftnMailpacket = require('../ftn_mail_packet.js'); +let ftnMailPacket = require('../ftn_mail_packet.js'); let ftnUtil = require('../ftn_util.js'); let Address = require('../ftn_address.js'); let Log = require('../logger.js').log; let moment = require('moment'); let _ = require('lodash'); +let paths = require('path'); +let mkdirp = require('mkdirp'); exports.moduleInfo = { name : 'FTN', @@ -18,75 +20,191 @@ exports.moduleInfo = { author : 'NuSkooler', }; +/* + :TODO: + * Add bundle timer (arcmail) + * Queue until time elapses / fixed time interval + * Pakcets append until >= max byte size + * [if arch type is not empty): Packets -> bundle until max byte size -> repeat process + * NetMail needs explicit isNetMail() check + * NetMail filename / location / etc. is still unknown - need to post on groups & get real answers +*/ + exports.getModule = FTNMessageScanTossModule; function FTNMessageScanTossModule() { MessageScanTossModule.call(this); if(_.has(Config, 'scannerTossers.ftn_bso')) { - this.config = Config.scannerTossers.ftn_bso; + this.moduleConfig = Config.scannerTossers.ftn_bso; } + + this.isDefaultDomainZone = function(networkName, address) { + return(networkName === this.moduleConfig.defaultNetwork && address.zone === this.moduleConfig.defaultZone); + } + + this.getOutgoingPacketDir = function(networkName, remoteAddress) { + let dir = this.moduleConfig.paths.outbound; + if(!this.isDefaultDomainZone(networkName, remoteAddress)) { + const hexZone = `000${remoteAddress.zone.toString(16)}`.substr(-3); + dir = paths.join(dir, `${networkName.toLowerCase()}.${hexZone}`); + } + return dir; + }; + + this.getOutgoingPacketFileName = function(basePath, message, isTemp) { + // + // Generating an outgoing packet file name comes with a few issues: + // * We must use DOS 8.3 filenames due to legacy systems that receive + // the packet not understanding LFNs + // * We need uniqueness; This is especially important with packets that + // end up in bundles and on the receiving/remote system where conflicts + // with other systems could also occur + // + // There are a lot of systems in use here for the name: + // * HEX CRC16/32 of data + // * HEX UNIX timestamp + // * Mystic at least at one point, used Hex8(day of month + seconds past midnight + hundredths of second) + // See https://groups.google.com/forum/#!searchin/alt.bbs.mystic/netmail$20filename/alt.bbs.mystic/m1xLnY8i1pU/YnG2excdl6MJ + // * SBBSEcho uses DDHHMMSS - see https://github.com/ftnapps/pkg-sbbs/blob/master/docs/fidonet.txt + // * We already have a system for 8-character serial number gernation that is + // used for e.g. in FTS-0009.001 MSGIDs... let's use that! + // + const name = ftnUtil.getMessageSerialNumber(message); + const ext = (true === isTemp) ? 'pk_' : 'pkt'; + return paths.join(basePath, `${name}.${ext}`); + }; - this.createMessagePacket = function(message, config) { - this.prepareMessage(message); + this.createMessagePacket = function(message, options) { + this.prepareMessage(message, options); let packet = new ftnMailPacket.Packet(); - let packetHeader = new ftnMailpacket.PacketHeader(); - packetHeader.init( - config.network.localAddress, - config.remoteAddress); + let packetHeader = new ftnMailPacket.PacketHeader( + options.network.localAddress, + options.remoteAddress, + options.nodeConfig.packetType); - packetHeader.setPassword(config.remoteNode.packetPassword || ''); + packetHeader.password = options.nodeConfig.packetPassword || ''; + + if(message.isPrivate()) { + // :TODO: this should actually be checking for isNetMail()!! + } else { + const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.remoteAddress); + + mkdirp(outgoingDir, err => { + if(err) { + // :TODO: Handle me!! + } else { + packet.write( + this.getOutgoingPacketFileName(outgoingDir, message), + packetHeader, + [ message ], + { encoding : options.encoding } + ); + } + }); + } + }; - this.prepareMessage = function(message, config) { + this.prepareMessage = function(message, options) { // // Set various FTN kludges/etc. // message.meta.FtnProperty = message.meta.FtnProperty || {}; - message.meta.FtnProperty.ftn_orig_node = config.network.localAddress.node; - message.meta.FtnProperty.ftn_dest_node = config.remoteAddress.node; - message.meta.FtnProperty.ftn_orig_network = config.network.localAddress.net; - message.meta.FtnProperty.ftn_dest_network = config.remoteAddress.net; + message.meta.FtnKludge = message.meta.FtnKludge || {}; + + message.meta.FtnProperty.ftn_orig_node = options.network.localAddress.node; + message.meta.FtnProperty.ftn_dest_node = options.remoteAddress.node; + message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net; + message.meta.FtnProperty.ftn_dest_network = options.remoteAddress.net; // :TODO: attr1 & 2 message.meta.FtnProperty.ftn_cost = 0; + + message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); - message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); - message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(config.network.localAddress); - + // :TODO: Need an explicit isNetMail() check if(message.isPrivate()) { // // NetMail messages need a FRL-1005.001 "Via" line // http://ftsc.org/docs/frl-1005.001 // - if(_.isString(message.meta.FtnKludge['Via'])) { - message.meta.FtnKludge['Via'] = [ message.meta.FtnKludge['Via'] ]; + if(_.isString(message.meta.FtnKludge.Via)) { + message.meta.FtnKludge.Via = [ message.meta.FtnKludge.Via ]; } - message.meta.FtnKludge['Via'] = message.meta.FtnKludge['Via'] || []; - message.meta.FtnKludge['Via'].push(ftnUtil.getVia(config.network.localAddress)); + message.meta.FtnKludge.Via = message.meta.FtnKludge.Via || []; + message.meta.FtnKludge.Via.push(ftnUtil.getVia(options.network.localAddress)); } else { - message.meta.FtnProperty.ftn_area = message.areaTag; + // + // EchoMail requires some additional properties & kludges + // + message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(options.network.localAddress); + message.meta.FtnProperty.ftn_area = Config.messageNetworks.ftn.areas[message.areaTag].tag; + + // + // When exporting messages, we should create/update SEEN-BY + // with remote address(s) we are exporting to. + // + message.meta.FtnProperty.ftn_seen_by = + ftnUtil.getUpdatedSeenByEntries( + message.meta.FtnProperty.ftn_seen_by, + Config.messageNetworks.ftn.areas[message.areaTag].uplinks + ); + + // + // And create/update PATH for ourself + // + message.meta.FtnKludge.PATH = + ftnUtil.getUpdatedPathEntries(message.meta.FtnKludge.PATH, options.network.localAddress); } - + // - // When exporting messages, we should create/update SEEN-BY - // with remote address(s) we are exporting to. + // Additional kludges + // + message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress); + message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); + + if(!message.meta.FtnKludge.PID) { + message.meta.FtnKludge.PID = ftnUtil.getProductIdentifier(); + } + + if(!message.meta.FtnKludge.TID) { + // :TODO: Create TID!! + //message.meta.FtnKludge.TID = + } + // - message.meta.FtnProperty.ftn_seen_by = - ftnUtil.getUpdatedSeenByEntries( - message.meta.FtnProperty.ftn_seen_by, - Config.messageNetworks.ftn.areas[message.areaTag].uplinks - ); - - // - // And create/update PATH for ourself - // - message.meta.FtnKludge['PATH'] = - ftnUtil.getUpdatedPathEntries( - message.meta.FtnKludge['PATH'], - config.network.localAddress.node - ); + // Determine CHRS and actual internal encoding name + // Try to preserve anything already here + let encoding = options.nodeConfig.encoding || 'utf8'; + if(message.meta.FtnKludge.CHRS) { + const encFromChars = ftnUtil.getEncodingFromCharacterSetIdentifier(message.meta.FtnKludge.CHRS); + if(encFromChars) { + encoding = encFromChars; + } + } + + options.encoding = encoding; // save for later + message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding); + // :TODO: FLAGS kludge? + // :TODO: Add REPLY kludge if appropriate + + }; + + + // :TODO: change to something like isAreaConfigValid + // check paths, Addresses, etc. + this.isAreaConfigComplete = function(areaConfig) { + if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) { + return false; + } + + if(_.isString(areaConfig.uplinks)) { + areaConfig.uplinks = areaConfig.uplinks.split(' '); + } + + return (_.isArray(areaConfig.uplinks)); }; } @@ -95,23 +213,25 @@ require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); FTNMessageScanTossModule.prototype.startup = function(cb) { Log.info('FidoNet Scanner/Tosser starting up'); - - cb(null); + + FTNMessageScanTossModule.super_.prototype.startup.call(this, cb); }; FTNMessageScanTossModule.prototype.shutdown = function(cb) { Log.info('FidoNet Scanner/Tosser shutting down'); - cb(null); + FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }; -FTNMessageScanTossModule.prototype.record = function(message, cb) { - if(!_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) { +FTNMessageScanTossModule.prototype.record = function(message) { + if(!_.has(this, 'moduleConfig.nodes') || + !_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) + { return; } - const area = Config.messageNetworks.ftn.areas[message.areaTag]; - if(!_.isString(area.ftnArea) || !_.isArray(area.uplinks)) { + const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag]; + if(!this.isAreaConfigComplete(areaConfig)) { // :TODO: should probably log a warning here return; } @@ -119,20 +239,31 @@ FTNMessageScanTossModule.prototype.record = function(message, cb) { // // For each uplink, find the best configuration match // - area.uplinks.forEach(uplink => { + areaConfig.uplinks.forEach(uplink => { // :TODO: sort by least # of '*' & take top? - let matchNodes = _.filter(Object.keys(Config.scannerTossers.ftn_bso.nodes), addr => { + const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { return Address.fromString(addr).isMatch(uplink); - }); - - if(matchNodes.length > 0) { - const nodeKey = matchNodes[0]; + })[0]; + if(nodeKey) { + const processOptions = { + nodeConfig : this.moduleConfig.nodes[nodeKey], + network : Config.messageNetworks.ftn.networks[areaConfig.network], + remoteAddress : Address.fromString(uplink), + networkName : areaConfig.network, + }; + + if(_.isString(processOptions.network.localAddress)) { + // :TODO: move/cache this - e.g. @ startup(). Think about due to Config cache + processOptions.network.localAddress = Address.fromString(processOptions.network.localAddress); + } + + // :TODO: Validate the rest of the matching config -- or do that elsewhere, e.g. startup() + + this.createMessagePacket(message, processOptions); } }); - - cb(null); // :TODO: should perhaps record in batches - e.g. start an event, record // to temp location until time is hit or N achieved such that if multiple diff --git a/mods/msg_area_post_fse.js b/mods/msg_area_post_fse.js index 2b0c488d..16292cea 100644 --- a/mods/msg_area_post_fse.js +++ b/mods/msg_area_post_fse.js @@ -1,12 +1,13 @@ /* jslint node: true */ 'use strict'; -var FullScreenEditorModule = require('../core/fse.js').FullScreenEditorModule; -var Message = require('../core/message.js').Message; -var user = require('../core/user.js'); +let FullScreenEditorModule = require('../core/fse.js').FullScreenEditorModule; +//var Message = require('../core/message.js').Message; +let persistMessage = require('../core/message_area.js').persistMessage; +let user = require('../core/user.js'); -var _ = require('lodash'); -var async = require('async'); +let _ = require('lodash'); +let async = require('async'); exports.getModule = AreaPostFSEModule; @@ -36,9 +37,12 @@ function AreaPostFSEModule(options) { }); }, function saveMessage(callback) { + persistMessage(msg, callback); + /* msg.persist(function persisted(err) { callback(err); }); + */ } ], function complete(err) { From 1417b7efdd27e30c39b3bdb06a4199b0fe4e86af Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 23 Feb 2016 21:56:22 -0700 Subject: [PATCH 08/27] * Fix messages with no origin line * Fix end of message/termination detection for FTN packets * Start of archive support -- one use will be FTN archives * Work on FTN ArcMail/bundles --- core/archive_util.js | 114 ++++++++++++++++++++++++++++++++ core/config.js | 22 ++++-- core/ftn_mail_packet.js | 46 ++++++------- core/message.js | 2 - core/scanner_tossers/ftn_bso.js | 84 ++++++++++++++++++++--- 5 files changed, 229 insertions(+), 39 deletions(-) create mode 100644 core/archive_util.js diff --git a/core/archive_util.js b/core/archive_util.js new file mode 100644 index 00000000..ba7d3c4e --- /dev/null +++ b/core/archive_util.js @@ -0,0 +1,114 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +let Config = require('./config.js').config; + +// base/modules +let fs = require('fs'); +let _ = require('lodash'); +let pty = require('ptyw.js'); + +module.exports = class ArchiveUtil { + + constructor() { + this.archivers = {}; + this.longestSignature = 0; + } + + init() { + // + // Load configuration + // + if(_.has(Config, 'archivers')) { + Object.keys(Config.archivers).forEach(archKey => { + const arch = Config.archivers[archKey]; + if(!_.isString(arch.sig) || + !_.isString(arch.compressCmd) || + !_.isString(arch.decompressCmd) || + !_.isArray(arch.compressArgs) || + !_.isArray(arch.decompressArgs)) + { + // :TODO: log warning + return; + } + + const archiver = { + compressCmd : arch.compressCmd, + compressArgs : arch.compressArgs, + decompressCmd : arch.decompressCmd, + decompressArgs : arch.decompressArgs, + sig : new Buffer(arch.sig, 'hex'), + offset : arch.offset || 0, + }; + + this.archivers[archKey] = archiver; + + if(archiver.offset + archiver.sig.length > this.longestSignature) { + this.longestSignature = archiver.offset + archiver.sig.length; + } + }); + } + } + + detectType(path, cb) { + fs.open(path, 'r', (err, fd) => { + if(err) { + cb(err); + return; + } + + let buf = new Buffer(this.longestSignature); + fs.read(fd, buf, 0, buf.length, 0, (err, bytesRead) => { + if(err) { + cb(err); + return; + } + + // return first match + const detected = _.find(this.archivers, arch => { + const lenNeeded = arch.offset + arch.sig.length; + + if(buf.length < lenNeeded) { + return false; + } + + const comp = buf.slice(arch.offset, arch.offset + arch.sig.length); + return (arch.sig.equals(comp)); + }); + + cb(detected ? null : new Error('Unknown type'), detected); + }); + }); + } + + compressTo(archType, archivePath, files, cb) { + const archiver = this.archivers[archType]; + if(!archiver) { + cb(new Error('Unknown archive type: ' + archType)); + return; + } + + let args = _.clone(archiver.compressArgs); // don't much with orig + for(let i = 0; i < args.length; ++i) { + args[i] = args[i].format({ + archivePath : archivePath, + fileList : files.join(' '), + }); + } + + let comp = pty.spawn(archiver.compressCmd, args, { + cols : 80, + rows : 24, + // :TODO: cwd + }); + + comp.on('exit', exitCode => { + cb(0 === exitCode ? null : new Error('Compression failed with exit code: ' + exitCode)); + }); + } + + extractTo(archivePath, extractPath, archType, cb) { + + } +} diff --git a/core/config.js b/core/config.js index 85865ee3..76521f97 100644 --- a/core/config.js +++ b/core/config.js @@ -209,6 +209,18 @@ function getDefaultConfig() { } }, + archivers : { + zip : { + name : "PKZip", + sig : "504b0304", + offset : 0, + compressCmd : "7z", + compressArgs : [ "a", "-tzip", "{archivePath}", "{fileList}" ], + decompressCmd : "7z", + decompressArgs : [ "e", "-o{extractDir}", "{archivePath}" ] + } + }, + messageConferences : { system_internal : { name : 'System Internal', @@ -231,13 +243,13 @@ function getDefaultConfig() { scannerTossers : { ftn_bso : { paths : { - outbound : paths.join(__dirname, './../mail/out/'), - inbound : paths.join(__dirname, './../mail/in/'), - secInbound : paths.join(__dirname, './../mail/secin/'), + outbound : paths.join(__dirname, './../mail/ftn_out/'), + inbound : paths.join(__dirname, './../mail/ftn_in/'), + secInbound : paths.join(__dirname, './../mail/ftn_secin/'), }, - maxPacketByteSize : 256000, - maxBundleByteSize : 256000, + maxPacketByteSize : 512000, // 512k, before placing messages in a new pkt + maxBundleByteSize : 2048000, // 2M, before creating another archive } }, diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 34801e6c..a0d1b8a3 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -419,36 +419,34 @@ function Packet() { // Decode |messageBodyBuffer| using |encoding| defaulted or detected above // // :TODO: Look into \xec thing more - document - const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/[\xec\n]/g, '').split(/\r/g); - let preOrigin = true; + const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/\xec/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g); + let endOfMessage = true; messageLines.forEach(line => { if(0 === line.length) { messageBodyData.message.push(''); return; } - - if(preOrigin) { - if(line.startsWith('AREA:')) { - messageBodyData.area = line.substring(line.indexOf(':') + 1).trim(); - } else if(line.startsWith('--- ')) { - // Tear Lines are tracked allowing for specialized display/etc. - messageBodyData.tearLine = line; - } else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) { // To spec is " * Origin: ..." - messageBodyData.originLine = line; - preOrigin = false; - } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { - addKludgeLine(line.slice(1)); - } else { - // regular ol' message line - messageBodyData.message.push(line); + + if(line.startsWith('AREA:')) { + messageBodyData.area = line.substring(line.indexOf(':') + 1).trim(); + } else if(line.startsWith('--- ')) { + // Tear Lines are tracked allowing for specialized display/etc. + messageBodyData.tearLine = line; + } else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) { // To spec is " * Origin: ..." + messageBodyData.originLine = line; + endOfMessage = false; // Anything past origin is not part of the message body + } else if(line.startsWith('SEEN-BY:')) { + endOfMessage = true; // Anything past the first SEEN-BY is not part of the message body + messageBodyData.seenBy.push(line.substring(line.indexOf(':') + 1).trim()); + } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { + if('PATH:' === line.slice(1, 6)) { + endOfMessage = true; // Anything pats the first PATH is not part of the message body } - } else { - if(line.startsWith('SEEN-BY:')) { - messageBodyData.seenBy.push(line.substring(line.indexOf(':') + 1).trim()); - } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { - addKludgeLine(line.slice(1)); - } + addKludgeLine(line.slice(1)); + } else if(endOfMessage) { + // regular ol' message line + messageBodyData.message.push(line); } }); @@ -486,7 +484,7 @@ function Packet() { .scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max .scan('message', NULL_TERM_BUFFER) .tap(function tapped(msgData) { - if(!msgData.ftn_orig_node) { + if(!msgData.messageType) { // end marker -- no more messages end(); return; diff --git a/core/message.js b/core/message.js index f506f91b..1a5ddd3b 100644 --- a/core/message.js +++ b/core/message.js @@ -110,8 +110,6 @@ Message.FtnPropertyNames = { FtnOrigNetwork : 'ftn_orig_network', FtnDestNetwork : 'ftn_dest_network', FtnAttrFlags : 'ftn_attr_flags', - //FtnAttrFlags1 : 'ftn_attr_flags1', - //FtnAttrFlags2 : 'ftn_attr_flags2', FtnCost : 'ftn_cost', FtnOrigZone : 'ftn_orig_zone', FtnDestZone : 'ftn_dest_zone', diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 71e95aae..fa211146 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -8,11 +8,14 @@ let ftnMailPacket = require('../ftn_mail_packet.js'); let ftnUtil = require('../ftn_util.js'); let Address = require('../ftn_address.js'); let Log = require('../logger.js').log; +let ArchiveUtil = require('../archive_util.js'); let moment = require('moment'); let _ = require('lodash'); let paths = require('path'); let mkdirp = require('mkdirp'); +let async = require('async'); +let fs = require('fs'); exports.moduleInfo = { name : 'FTN', @@ -35,6 +38,9 @@ exports.getModule = FTNMessageScanTossModule; function FTNMessageScanTossModule() { MessageScanTossModule.call(this); + this.archUtil = new ArchiveUtil(); + this.archUtil.init(); + if(_.has(Config, 'scannerTossers.ftn_bso')) { this.moduleConfig = Config.scannerTossers.ftn_bso; } @@ -43,10 +49,10 @@ function FTNMessageScanTossModule() { return(networkName === this.moduleConfig.defaultNetwork && address.zone === this.moduleConfig.defaultZone); } - this.getOutgoingPacketDir = function(networkName, remoteAddress) { + this.getOutgoingPacketDir = function(networkName, destAddress) { let dir = this.moduleConfig.paths.outbound; - if(!this.isDefaultDomainZone(networkName, remoteAddress)) { - const hexZone = `000${remoteAddress.zone.toString(16)}`.substr(-3); + if(!this.isDefaultDomainZone(networkName, destAddress)) { + const hexZone = `000${destAddress.zone.toString(16)}`.substr(-3); dir = paths.join(dir, `${networkName.toLowerCase()}.${hexZone}`); } return dir; @@ -75,6 +81,59 @@ function FTNMessageScanTossModule() { return paths.join(basePath, `${name}.${ext}`); }; + this.getOutgoingFlowFileName = function(basePath, destAddress, exportType, extSuffix) { + if(destAddress.point) { + + } else { + // + // Use |destAddress| nnnnNNNN.??? where nnnn is dest net and NNNN is dest + // node. This seems to match what Mystic does + // + return `${Math.abs(destAddress.net)}${Math.abs(destAddress.node)}.${exportType[1]}${extSuffix}`; + } + }; + + this.getOutgoingBundleFileName = function(basePath, sourceAddress, destAddress, cb) { + // + // Base filename is constructed as such: + // * If this |destAddress| is *not* a point address, we use NNNNnnnn where + // NNNN is 0 padded hex of dest net - source net and and nnnn is 0 padded + // hex of dest node - source node. + // * If |destAddress| is a point, NNNN becomes 0000 and nnnn becomes 'p' + + // 3 digit 0 padded hex point + // + // Extension is dd? where dd is Su...Mo and ? is 0...Z as collisions arise + // + var basename; + if(destAddress.point) { + const pointHex = `000${destAddress.point}`.substr(-3); + basename = `0000p${pointHex}`; + } else { + basename = + `0000${Math.abs(sourceAddress.net - destAddress.net).toString(16)}`.substr(-4) + + `0000${Math.abs(sourceAddress.node - destAddress.node).toString(16)}`.substr(-4); + } + + // + // We need to now find the first entry that does not exist starting + // with dd0 to ddz + // + const EXT_SUFFIXES = '0123456789abcdefghijklmnopqrstuvwxyz'.split(''); + let fileName = `${basename}.${moment().format('dd').toLowerCase()}`; + async.detectSeries(EXT_SUFFIXES, (suffix, callback) => { + const checkFileName = fileName + suffix; + fs.stat(paths.join(basePath, checkFileName), (err, stats) => { + callback((err && 'ENOENT' === err.code) ? true : false); + }); + }, finalSuffix => { + if(finalSuffix) { + cb(null, paths.join(basePath, fileName + finalSuffix)); + } else { + cb(new Error('Could not acquire a bundle filename!')); + } + }); + }; + this.createMessagePacket = function(message, options) { this.prepareMessage(message, options); @@ -82,7 +141,7 @@ function FTNMessageScanTossModule() { let packetHeader = new ftnMailPacket.PacketHeader( options.network.localAddress, - options.remoteAddress, + options.destAddress, options.nodeConfig.packetType); packetHeader.password = options.nodeConfig.packetPassword || ''; @@ -90,12 +149,15 @@ function FTNMessageScanTossModule() { if(message.isPrivate()) { // :TODO: this should actually be checking for isNetMail()!! } else { - const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.remoteAddress); + const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress); mkdirp(outgoingDir, err => { if(err) { // :TODO: Handle me!! } else { + this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => { + console.log(path); + }); packet.write( this.getOutgoingPacketFileName(outgoingDir, message), packetHeader, @@ -116,16 +178,20 @@ function FTNMessageScanTossModule() { message.meta.FtnKludge = message.meta.FtnKludge || {}; message.meta.FtnProperty.ftn_orig_node = options.network.localAddress.node; - message.meta.FtnProperty.ftn_dest_node = options.remoteAddress.node; + message.meta.FtnProperty.ftn_dest_node = options.destAddress.node; message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net; - message.meta.FtnProperty.ftn_dest_network = options.remoteAddress.net; + message.meta.FtnProperty.ftn_dest_network = options.destAddress.net; // :TODO: attr1 & 2 message.meta.FtnProperty.ftn_cost = 0; message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); // :TODO: Need an explicit isNetMail() check + let ftnAttribute = 0; + if(message.isPrivate()) { + ftnAttribute |= ftnMailPacket.Packet.Attribute.Private; + // // NetMail messages need a FRL-1005.001 "Via" line // http://ftsc.org/docs/frl-1005.001 @@ -159,6 +225,8 @@ function FTNMessageScanTossModule() { ftnUtil.getUpdatedPathEntries(message.meta.FtnKludge.PATH, options.network.localAddress); } + message.meta.FtnProperty.ftn_attr_flags = ftnAttribute; + // // Additional kludges // @@ -249,7 +317,7 @@ FTNMessageScanTossModule.prototype.record = function(message) { const processOptions = { nodeConfig : this.moduleConfig.nodes[nodeKey], network : Config.messageNetworks.ftn.networks[areaConfig.network], - remoteAddress : Address.fromString(uplink), + destAddress : Address.fromString(uplink), networkName : areaConfig.network, }; From ae20dc1f7cafde1e472351ef1dcc3ea21ce364e1 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 23 Feb 2016 23:38:05 -0700 Subject: [PATCH 09/27] * Fix FTN packet created date/time & moment stuff --- core/ftn_mail_packet.js | 54 +++++++++++++++++++-------------- core/scanner_tossers/ftn_bso.js | 6 ++-- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index a0d1b8a3..891ceaca 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -39,7 +39,7 @@ const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; class PacketHeader { - constructor(origAddr, destAddr, version, created) { + constructor(origAddr, destAddr, version, createdMoment) { const EMPTY_ADDRESS = { node : 0, net : 0, @@ -51,18 +51,21 @@ class PacketHeader { this.origAddress = origAddr || EMPTY_ADDRESS; this.destAddress = destAddr || EMPTY_ADDRESS; - this.created = created || moment(); + this.created = createdMoment || moment(); // uncommon to set the following explicitly - this.prodCodeLo = 0xfe; // http://ftsc.org/docs/fta-1005.003 - this.prodRevLo = 0; - this.baud = 0; - this.packetType = FTN_PACKET_HEADER_TYPE; - this.password = ''; - this.prodData = 0x47694e45; // "ENiG" + this.prodCodeLo = 0xfe; // http://ftsc.org/docs/fta-1005.003 + this.prodRevLo = 0; + this.baud = 0; + this.packetType = FTN_PACKET_HEADER_TYPE; + this.password = ''; + this.prodData = 0x47694e45; // "ENiG" this.capWord = 0x0001; this.capWordValidate = ((this.capWord & 0xff) << 8) | ((this.capWord >> 8) & 0xff); + + this.prodCodeHi = 0xfe; // see above + this.prodRevHi = 0; } get origAddress() { @@ -129,17 +132,24 @@ class PacketHeader { } get created() { - return moment(this); // use year, month, etc. properties + return moment({ + year : this.year, + month : this.month - 1, // moment uses 0 indexed months + date : this.day, + hour : this.hour, + minute : this.minute, + second : this.second + }); } set created(momentCreated) { if(!moment.isMoment(momentCreated)) { - created = moment(momentCreated); + momentCreated = moment(momentCreated); } this.year = momentCreated.year(); - this.month = momentCreated.month(); - this.day = momentCreated.date(); // day of month + this.month = momentCreated.month() + 1; // moment uses 0 indexed months + this.day = momentCreated.date(); // day of month this.hour = momentCreated.hour(); this.minute = momentCreated.minute(); this.second = momentCreated.second(); @@ -254,20 +264,19 @@ function Packet() { // :TODO: should fill bytes be 0? } } - + packetHeader.created = moment({ year : packetHeader.year, - month : packetHeader.month, + month : packetHeader.month - 1, // moment uses 0 indexed months date : packetHeader.day, hour : packetHeader.hour, minute : packetHeader.minute, second : packetHeader.second }); - + let ph = new PacketHeader(); _.assign(ph, packetHeader); - cb(null, ph); }); }; @@ -277,12 +286,13 @@ function Packet() { buffer.writeUInt16LE(packetHeader.origNode, 0); buffer.writeUInt16LE(packetHeader.destNode, 2); - buffer.writeUInt16LE(packetHeader.created.year(), 4); - buffer.writeUInt16LE(packetHeader.created.month(), 6); - buffer.writeUInt16LE(packetHeader.created.date(), 8); // day of month - buffer.writeUInt16LE(packetHeader.created.hour(), 10); - buffer.writeUInt16LE(packetHeader.created.minute(), 12); - buffer.writeUInt16LE(packetHeader.created.second(), 14); + buffer.writeUInt16LE(packetHeader.year, 4); + buffer.writeUInt16LE(packetHeader.month, 6); + buffer.writeUInt16LE(packetHeader.day, 8); + buffer.writeUInt16LE(packetHeader.hour, 10); + buffer.writeUInt16LE(packetHeader.minute, 12); + buffer.writeUInt16LE(packetHeader.second, 14); + buffer.writeUInt16LE(packetHeader.baud, 16); buffer.writeUInt16LE(FTN_PACKET_HEADER_TYPE, 18); buffer.writeUInt16LE(packetHeader.origNet, 20); diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index fa211146..cd71358d 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -212,11 +212,9 @@ function FTNMessageScanTossModule() { // When exporting messages, we should create/update SEEN-BY // with remote address(s) we are exporting to. // + const seenByAdditions = [ options.network.localAddress ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks); message.meta.FtnProperty.ftn_seen_by = - ftnUtil.getUpdatedSeenByEntries( - message.meta.FtnProperty.ftn_seen_by, - Config.messageNetworks.ftn.areas[message.areaTag].uplinks - ); + ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions); // // And create/update PATH for ourself From 76bbc43600f1d73837c1ece274d75beed70f4c7f Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 28 Feb 2016 22:04:03 -0700 Subject: [PATCH 10/27] * Start work on FTN/BSO schedule via later.js * Utilize last scan message ID to scan areas * Lots of changes to FTN packet creation * Create packets with target max size * Create ArcMail bundles when configured to do so --- core/archive_util.js | 13 +- core/config.js | 12 +- core/database.js | 10 + core/ftn_mail_packet.js | 155 ++++++++++++ core/ftn_util.js | 6 +- core/scanner_tossers/ftn_bso.js | 423 ++++++++++++++++++++++++++++++-- package.json | 3 +- 7 files changed, 595 insertions(+), 27 deletions(-) diff --git a/core/archive_util.js b/core/archive_util.js index ba7d3c4e..cbae6d13 100644 --- a/core/archive_util.js +++ b/core/archive_util.js @@ -50,6 +50,15 @@ module.exports = class ArchiveUtil { }); } } + + haveArchiver(archType) { + if(!archType) { + return false; + } + + archType = archType.toLowerCase(); + return archType in this.archivers; + } detectType(path, cb) { fs.open(path, 'r', (err, fd) => { @@ -83,7 +92,9 @@ module.exports = class ArchiveUtil { } compressTo(archType, archivePath, files, cb) { + archType = archType.toLowerCase(); const archiver = this.archivers[archType]; + if(!archiver) { cb(new Error('Unknown archive type: ' + archType)); return; @@ -104,7 +115,7 @@ module.exports = class ArchiveUtil { }); comp.on('exit', exitCode => { - cb(0 === exitCode ? null : new Error('Compression failed with exit code: ' + exitCode)); + cb(exitCode ? new Error('Compression failed with exit code: ' + exitCode) : null); }); } diff --git a/core/config.js b/core/config.js index 76521f97..47b0003d 100644 --- a/core/config.js +++ b/core/config.js @@ -211,7 +211,7 @@ function getDefaultConfig() { archivers : { zip : { - name : "PKZip", + name : "PKZip", // :TODO: Use key for this sig : "504b0304", offset : 0, compressCmd : "7z", @@ -246,10 +246,16 @@ function getDefaultConfig() { outbound : paths.join(__dirname, './../mail/ftn_out/'), inbound : paths.join(__dirname, './../mail/ftn_in/'), secInbound : paths.join(__dirname, './../mail/ftn_secin/'), + temp : paths.join(__dirname, './../mail/ftn_temp'), }, - maxPacketByteSize : 512000, // 512k, before placing messages in a new pkt - maxBundleByteSize : 2048000, // 2M, before creating another archive + // + // Packet and (ArcMail) bundle target sizes are just that: targets. + // Actual sizes may be slightly larger when we must place a full + // PKT contents *somewhere* + // + packetTargetByteSize : 512000, // 512k, before placing messages in a new pkt + bundleTargetByteSize : 2048000, // 2M, before creating another archive } }, diff --git a/core/database.js b/core/database.js index cb1a97f4..3e3b6faf 100644 --- a/core/database.js +++ b/core/database.js @@ -204,6 +204,7 @@ function createMessageBaseTables() { ');' ); + // :TODO: Not currently used dbs.message.run( 'CREATE TABLE IF NOT EXISTS user_message_status (' + ' user_id INTEGER NOT NULL,' + @@ -213,6 +214,15 @@ function createMessageBaseTables() { ' FOREIGN KEY(user_id) REFERENCES user(id)' + ');' ); + + dbs.message.run( + `CREATE TABLE IF NOT EXISTS message_area_last_scan ( + scan_toss VARCHAR NOT NULL, + area_tag VARCHAR NOT NULL, + message_id INTEGER NOT NULL, + UNIQUE(scan_toss, area_tag) + );` + ); } function createInitialMessageValues() { diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 891ceaca..ecc0f1c8 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -280,6 +280,44 @@ function Packet() { cb(null, ph); }); }; + + this.getPacketHeaderBuffer = function(packetHeader) { + let buffer = new Buffer(FTN_PACKET_HEADER_SIZE); + + buffer.writeUInt16LE(packetHeader.origNode, 0); + buffer.writeUInt16LE(packetHeader.destNode, 2); + buffer.writeUInt16LE(packetHeader.year, 4); + buffer.writeUInt16LE(packetHeader.month, 6); + buffer.writeUInt16LE(packetHeader.day, 8); + buffer.writeUInt16LE(packetHeader.hour, 10); + buffer.writeUInt16LE(packetHeader.minute, 12); + buffer.writeUInt16LE(packetHeader.second, 14); + + buffer.writeUInt16LE(packetHeader.baud, 16); + buffer.writeUInt16LE(FTN_PACKET_HEADER_TYPE, 18); + buffer.writeUInt16LE(packetHeader.origNet, 20); + buffer.writeUInt16LE(packetHeader.destNet, 22); + buffer.writeUInt8(packetHeader.prodCodeLo, 24); + buffer.writeUInt8(packetHeader.prodRevHi, 25); + + const pass = ftn.stringToNullPaddedBuffer(packetHeader.password, 8); + pass.copy(buffer, 26); + + buffer.writeUInt16LE(packetHeader.origZone, 34); + buffer.writeUInt16LE(packetHeader.destZone, 36); + buffer.writeUInt16LE(packetHeader.auxNet, 38); + buffer.writeUInt16LE(packetHeader.capWordValidate, 40); + buffer.writeUInt8(packetHeader.prodCodeHi, 42); + buffer.writeUInt8(packetHeader.prodRevLo, 43); + buffer.writeUInt16LE(packetHeader.capWord, 44); + buffer.writeUInt16LE(packetHeader.origZone2, 46); + buffer.writeUInt16LE(packetHeader.destZone2, 48); + buffer.writeUInt16LE(packetHeader.origPoint, 50); + buffer.writeUInt16LE(packetHeader.destPoint, 52); + buffer.writeUInt32LE(packetHeader.prodData, 54); + + return buffer; + } this.writePacketHeader = function(packetHeader, ws) { let buffer = new Buffer(FTN_PACKET_HEADER_SIZE); @@ -317,6 +355,8 @@ function Packet() { buffer.writeUInt32LE(packetHeader.prodData, 54); ws.write(buffer); + + return buffer.length; }; this.processMessageBody = function(messageBodyBuffer, cb) { @@ -577,6 +617,103 @@ function Packet() { }); }); }; + + this.getMessageEntryBuffer = function(message, options) { + let basicHeader = new Buffer(34); + + basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); + + const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); + dateTimeBuffer.copy(basicHeader, 14); + + // toUserName & fromUserName: up to 36 bytes in length, NULL term'd + // :TODO: DRY... + let toUserNameBuf = iconv.encode(message.toUserName + '\0', 'CP437').slice(0, 36); + toUserNameBuf[toUserNameBuf.length - 1] = '\0'; // ensure it's null term'd + + let fromUserNameBuf = iconv.encode(message.fromUserName + '\0', 'CP437').slice(0, 36); + fromUserNameBuf[fromUserNameBuf.length - 1] = '\0'; // ensure it's null term'd + + // subject: up to 72 bytes in length, NULL term'd + let subjectBuf = iconv.encode(message.subject + '\0', 'CP437').slice(0, 72); + subjectBuf[subjectBuf.length - 1] = '\0'; // ensure it's null term'd + + // + // message: unbound length, NULL term'd + // + // We need to build in various special lines - kludges, area, + // seen-by, etc. + // + // :TODO: Put this in it's own method + let msgBody = ''; + + function appendMeta(k, m) { + if(m) { + let a = m; + if(!_.isArray(a)) { + a = [ a ]; + } + a.forEach(v => { + msgBody += `${k}: ${v}\r`; + }); + } + } + + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // AREA:CONFERENCE + // Should be first line in a message + // + if(message.meta.FtnProperty.ftn_area) { + msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\r`; // note: no ^A (0x01) + } + + Object.keys(message.meta.FtnKludge).forEach(k => { + // we want PATH to be last + if('PATH' !== k) { + appendMeta(`\x01${k}`, message.meta.FtnKludge[k]); + } + }); + + msgBody += message.message + '\r'; + + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // Tear line should be near the bottom of a message + // + if(message.meta.FtnProperty.ftn_tear_line) { + msgBody += `${message.meta.FtnProperty.ftn_tear_line}\r`; + } + + // + // Origin line should be near the bottom of a message + // + if(message.meta.FtnProperty.ftn_origin) { + msgBody += `${message.meta.FtnProperty.ftn_origin}\r`; + } + + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // SEEN-BY and PATH should be the last lines of a message + // + appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); // note: no ^A (0x01) + + appendMeta('\x01PATH', message.meta.FtnKludge['PATH']); + + return Buffer.concat( [ + basicHeader, + toUserNameBuf, + fromUserNameBuf, + subjectBuf, + iconv.encode(msgBody + '\0', options.encoding) + ]); + }; this.writeMessage = function(message, ws, options) { let basicHeader = new Buffer(34); @@ -750,6 +887,24 @@ Packet.prototype.read = function(pathOrBuffer, iterator, cb) { ); }; +Packet.prototype.writeHeader = function(ws, packetHeader) { + return this.writePacketHeader(packetHeader, ws); +}; + +Packet.prototype.writeMessage = function(ws, message, options) { + +} + +Packet.prototype.writeMessageEntry = function(ws, msgEntry) { + ws.write(msgEntry); + return msgEntry.length; +}; + +Packet.prototype.writeTerminator = function(ws) { + ws.write(new Buffer( [ 0 ] )); // final extra null term + return 1; +}; + Packet.prototype.writeStream = function(ws, messages, options) { if(!_.isBoolean(options.terminatePacket)) { options.terminatePacket = true; diff --git a/core/ftn_util.js b/core/ftn_util.js index 5172b218..3a347b2b 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -138,9 +138,9 @@ function createMessageUuid(ftnMsgId, ftnArea) { return uuid.unparse(u); // to string } -function getMessageSerialNumber(message) { +function getMessageSerialNumber(messageId) { const msSinceEnigmaEpoc = (Date.now() - Date.UTC(2016, 1, 1)); - const hash = Math.abs(new FNV1a(msSinceEnigmaEpoc + message.messageId).value).toString(16); + const hash = Math.abs(new FNV1a(msSinceEnigmaEpoc + messageId).value).toString(16); return `00000000${hash}`.substr(-8); // return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + // message.messageId)).toString(16)).substr(-8); @@ -183,7 +183,7 @@ function getMessageSerialNumber(message) { // function getMessageIdentifier(message, address) { const addrStr = new Address(address).toString('5D'); - return `${message.messageId}.${message.areaTag.toLowerCase()}@${addrStr} ${getMessageSerialNumber(message)}`; + return `${message.messageId}.${message.areaTag.toLowerCase()}@${addrStr} ${getMessageSerialNumber(message.messageId)}`; } // diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index cd71358d..6f45fdb8 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -9,6 +9,8 @@ let ftnUtil = require('../ftn_util.js'); let Address = require('../ftn_address.js'); let Log = require('../logger.js').log; let ArchiveUtil = require('../archive_util.js'); +let msgDb = require('../database.js').dbs.message; +let Message = require('../message.js'); let moment = require('moment'); let _ = require('lodash'); @@ -16,10 +18,11 @@ let paths = require('path'); let mkdirp = require('mkdirp'); let async = require('async'); let fs = require('fs'); +let later = require('later'); exports.moduleInfo = { - name : 'FTN', - desc : 'FidoNet Style Message Scanner/Tosser', + name : 'FTN BSO', + desc : 'BSO style message scanner/tosser for FTN networks', author : 'NuSkooler', }; @@ -35,14 +38,21 @@ exports.moduleInfo = { exports.getModule = FTNMessageScanTossModule; +const SCHEDULE_REGEXP = /(?:^|or )?(@watch\:|@immediate)([^\0]+)?$/; + function FTNMessageScanTossModule() { MessageScanTossModule.call(this); + + let self = this; this.archUtil = new ArchiveUtil(); this.archUtil.init(); + if(_.has(Config, 'scannerTossers.ftn_bso')) { this.moduleConfig = Config.scannerTossers.ftn_bso; + + } this.isDefaultDomainZone = function(networkName, address) { @@ -58,7 +68,7 @@ function FTNMessageScanTossModule() { return dir; }; - this.getOutgoingPacketFileName = function(basePath, message, isTemp) { + this.getOutgoingPacketFileName = function(basePath, messageId, isTemp) { // // Generating an outgoing packet file name comes with a few issues: // * We must use DOS 8.3 filenames due to legacy systems that receive @@ -76,7 +86,7 @@ function FTNMessageScanTossModule() { // * We already have a system for 8-character serial number gernation that is // used for e.g. in FTS-0009.001 MSGIDs... let's use that! // - const name = ftnUtil.getMessageSerialNumber(message); + const name = ftnUtil.getMessageSerialNumber(messageId); const ext = (true === isTemp) ? 'pk_' : 'pkt'; return paths.join(basePath, `${name}.${ext}`); }; @@ -133,8 +143,8 @@ function FTNMessageScanTossModule() { } }); }; - - this.createMessagePacket = function(message, options) { + + this.exportMessage = function(message, options, cb) { this.prepareMessage(message, options); let packet = new ftnMailPacket.Packet(); @@ -153,18 +163,17 @@ function FTNMessageScanTossModule() { mkdirp(outgoingDir, err => { if(err) { - // :TODO: Handle me!! - } else { - this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => { + return cb(err); + } + this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => { console.log(path); - }); - packet.write( - this.getOutgoingPacketFileName(outgoingDir, message), - packetHeader, - [ message ], - { encoding : options.encoding } - ); - } + }); + packet.write( + this.getOutgoingPacketFileName(outgoingDir, message), + packetHeader, + [ message ], + { encoding : options.encoding } + ); }); } @@ -261,7 +270,7 @@ function FTNMessageScanTossModule() { // :TODO: change to something like isAreaConfigValid // check paths, Addresses, etc. - this.isAreaConfigComplete = function(areaConfig) { + this.isAreaConfigValid = function(areaConfig) { if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) { return false; } @@ -272,7 +281,291 @@ function FTNMessageScanTossModule() { return (_.isArray(areaConfig.uplinks)); }; + + + this.hasValidConfiguration = function() { + if(!_.has(this, 'moduleConfig.nodes') || !_.has(Config, 'messageNetworks.ftn.areas')) { + return false; + } + + return true; + }; + + this.parseScheduleString = function(schedStr) { + let schedule = {}; + + const m = SCHEDULE_REGEXP.exec(schedStr); + if(m) { + schedStr = schedStr.substr(0, m.index).trim(); + + if('@watch:' === m[1]) { + schedule.watchFile = m[2]; + } else if('@immediate' === m[1]) { + schedule.immediate = true; + } + } + if(schedStr.length > 0) { + const sched = later.parse.text(schedStr); + if(-1 === sched.error) { + schedule.sched = sched; + } + } + + // return undefined if we couldn't parse out anything useful + if(!_.isEmpty(schedule)) { + return schedule; + } + }; + + this.performImport = function() { + }; + + this.getAreaLastScanId = function(areaTag, cb) { + const sql = + `SELECT area_tag, message_id + FROM message_area_last_scan + WHERE scan_toss = "ftn_bso" AND area_tag = ? + LIMIT 1;`; + + msgDb.get(sql, [ areaTag ], (err, row) => { + cb(err, row ? row.message_id : 0); + }); + }; + + this.getNodeConfigKeyForUplink = function(uplink) { + // :TODO: sort by least # of '*' & take top? + const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { + return Address.fromString(addr).isMatch(uplink); + })[0]; + + return nodeKey; + }; + + this.exportMessagesByUuid = function(messageUuids, exportOpts, cb) { + // + // This method has a lot of madness going on: + // - Try to stuff messages into packets until we've hit the target size + // - We need to wait for write streams to finish before proceeding in many cases + // or data will be cut off when closing and creating a new stream + // + let exportedFiles = []; + let currPacketSize = self.moduleConfig.packetTargetByteSize; + let packet; + let ws; + let remainMessageBuf; + let remainMessageId; + + async.each(messageUuids, (msgUuid, nextUuid) => { + let message = new Message(); + + async.series( + [ + function finalizePrevious(callback) { + if(packet && currPacketSize >= self.moduleConfig.packetTargetByteSize) { + packet.writeTerminator(ws); + ws.end(); + ws.once('finish', () => { + callback(null); + }); + } else { + callback(null); + } + }, + function loadMessage(callback) { + message.load( { uuid : msgUuid }, err => { + if(!err) { + self.prepareMessage(message, exportOpts); + } + callback(err); + }); + }, + function createNewPacket(callback) { + if(currPacketSize >= self.moduleConfig.packetTargetByteSize) { + packet = new ftnMailPacket.Packet(); + + const packetHeader = new ftnMailPacket.PacketHeader( + exportOpts.network.localAddress, + exportOpts.destAddress, + exportOpts.nodeConfig.packetType); + + packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; + + // use current message ID for filename seed + const pktFileName = self.getOutgoingPacketFileName(exportOpts.exportDir, message.messageId); + exportedFiles.push(pktFileName); + + ws = fs.createWriteStream(pktFileName); + + currPacketSize = packet.writeHeader(ws, packetHeader); + + if(remainMessageBuf) { + currPacketSize += packet.writeMessageEntry(ws, remainMessageBuf); + remainMessageBuf = null; + } + } + + callback(null); + }, + function appendMessage(callback) { + const msgBuf = packet.getMessageEntryBuffer(message, exportOpts); + currPacketSize += msgBuf.length; + + if(currPacketSize >= self.moduleConfig.packetTargetByteSize) { + remainMessageBuf = msgBuf; // save for next packet + remainMessageId = message.messageId; + } else { + ws.write(msgBuf); + } + callback(null); + } + ], + err => { + nextUuid(err); + } + ); + }, err => { + if(err) { + cb(err); + } else { + async.series( + [ + function terminateLast(callback) { + if(packet) { + packet.writeTerminator(ws); + ws.end(); + ws.once('finish', () => { + callback(null); + }); + } else { + callback(null); + } + }, + function writeRemainPacket(callback) { + if(remainMessageBuf) { + // :TODO: DRY this with the code above -- they are basically identical + packet = new ftnMailPacket.Packet(); + + const packetHeader = new ftnMailPacket.PacketHeader( + exportOpts.network.localAddress, + exportOpts.destAddress, + exportOpts.nodeConfig.packetType); + + packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; + + // use current message ID for filename seed + const pktFileName = self.getOutgoingPacketFileName(exportOpts.exportDir, remainMessageId); + exportedFiles.push(pktFileName); + + ws = fs.createWriteStream(pktFileName); + + packet.writeHeader(ws, packetHeader); + ws.write(remainMessageBuf); + packet.writeTerminator(ws); + ws.end(); + ws.once('finish', () => { + callback(null); + }); + } else { + callback(null); + } + } + ], + err => { + cb(err, exportedFiles); + } + ); + } + }); + }; + + this.exportMessagesToUplinks = function(messageUuids, areaConfig, cb) { + async.each(areaConfig.uplinks, (uplink, nextUplink) => { + const nodeConfigKey = self.getNodeConfigKeyForUplink(uplink); + if(!nodeConfigKey) { + return nextUplink(); + } + + const exportOpts = { + nodeConfig : self.moduleConfig.nodes[nodeConfigKey], + network : Config.messageNetworks.ftn.networks[areaConfig.network], + destAddress : Address.fromString(uplink), + networkName : areaConfig.network, + exportDir : self.moduleConfig.paths.temp, + }; + + if(_.isString(exportOpts.network.localAddress)) { + exportOpts.network.localAddress = Address.fromString(exportOpts.network.localAddress); + } + + const outgoingDir = self.getOutgoingPacketDir(exportOpts.networkName, exportOpts.destAddress); + + async.waterfall( + [ + function createTempDir(callback) { + mkdirp(exportOpts.exportDir, err => { + callback(err); + }); + }, + function createOutgoingDir(callback) { + mkdirp(outgoingDir, err => { + callback(err); + }); + }, + function exportToTempArea(callback) { + self.exportMessagesByUuid(messageUuids, exportOpts, callback); + }, + function createArcMailBundle(exportedFileNames, callback) { + if(self.archUtil.haveArchiver(exportOpts.nodeConfig.archiveType)) { + // :TODO: support bundleTargetByteSize: + // + // Compress to a temp location then we'll move it in the next step + // + // Note that we must use the *final* output dir for getOutgoingBundleFileName() + // as it checks for collisions in bundle names! + // + self.getOutgoingBundleFileName(outgoingDir, exportOpts.network.localAddress, exportOpts.destAddress, (err, bundlePath) => { + if(err) { + return callback(err); + } + + // adjust back to temp path + const tempBundlePath = paths.join(exportOpts.exportDir, paths.basename(bundlePath)); + + self.archUtil.compressTo( + exportOpts.nodeConfig.archiveType, + tempBundlePath, + exportedFileNames, err => { + // :TODO: we need to delete the original input file(s) + fs.rename(tempBundlePath, bundlePath, err => { + callback(err, [ bundlePath ] ); + }); + } + ); + }); + } else { + callback(null, exportedFileNames); + } + }, + function moveFilesToOutgoing(exportedFileNames, callback) { + async.each(exportedFileNames, (oldPath, nextFile) => { + const ext = paths.extname(oldPath); + if('.pk_' === ext) { + const newPath = paths.join(outgoingDir, paths.basename(oldPath, ext) + '.pkt'); + fs.rename(oldPath, newPath, nextFile); + } else { + const newPath = paths.join(outgoingDir, paths.basename(oldPath)); + fs.rename(oldPath, newPath, nextFile); + } + }, callback); + } + ], + err => { + nextUplink(); + } + ); + }, cb); // complete + }; } require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); @@ -280,16 +573,107 @@ require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); FTNMessageScanTossModule.prototype.startup = function(cb) { Log.info('FidoNet Scanner/Tosser starting up'); + if(_.isObject(this.moduleConfig.schedule)) { + const exportSchedule = this.parseScheduleString(this.moduleConfig.schedule.export); + if(exportSchedule) { + if(exportSchedule.sched) { + let exporting = false; + this.exportTimer = later.setInterval( () => { + if(!exporting) { + exporting = true; + + Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message export...'); + + this.performExport(err => { + exporting = false; + }); + } + }, exportSchedule.sched); + } + + if(exportSchedule.watchFile) { + // :TODO: monitor file for changes/existance with gaze + } + } + } + FTNMessageScanTossModule.super_.prototype.startup.call(this, cb); }; FTNMessageScanTossModule.prototype.shutdown = function(cb) { Log.info('FidoNet Scanner/Tosser shutting down'); + + if(this.exportTimer) { + this.exportTimer.clear(); + } FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }; +FTNMessageScanTossModule.prototype.performExport = function(cb) { + // + // We're only concerned with areas related to FTN. For each area, loop though + // and let's find out what messages need exported. + // + if(!this.hasValidConfiguration()) { + return cb(new Error('No valid configurations for export')); + } + + // :TODO: Block exporting (e.g. ignore timer) until export is finished + + const getNewUuidsSql = + `SELECT message_uuid + FROM message + WHERE area_tag = ? AND message_id > ? + ORDER BY message_id;`; + + var self = this; + + async.each(Object.keys(Config.messageNetworks.ftn.areas), (areaTag, nextArea) => { + const areaConfig = Config.messageNetworks.ftn.areas[areaTag]; + if(!this.isAreaConfigValid(areaConfig)) { + return nextArea(); + } + + // + // For each message that is newer than that of the last scan + // we need to export to each configured associated uplink(s) + // + async.waterfall( + [ + function getLastScanId(callback) { + self.getAreaLastScanId(areaTag, callback); + }, + function getNewUuids(lastScanId, callback) { + msgDb.all(getNewUuidsSql, [ areaTag, lastScanId ], (err, rows) => { + if(err) { + callback(err); + } else { + callback(null, rows.map(r => r.message_uuid)); // convert to simple array of UUIDs + } + }); + }, + function exportToConfiguredUplinks(msgUuids, callback) { + self.exportMessagesToUplinks(msgUuids, areaConfig, err => { + // :TODO: Log/handle err + callback(null, msgUuids[msgUuids.length - 1]); + }); + }, + function updateLastScanId(newLastScanId, callback) { + callback(null); + } + ], + function complete(err) { + nextArea(); + } + ); + }, err => { + cb(err); + }); +}; + FTNMessageScanTossModule.prototype.record = function(message) { + /* if(!_.has(this, 'moduleConfig.nodes') || !_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) { @@ -297,7 +681,7 @@ FTNMessageScanTossModule.prototype.record = function(message) { } const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag]; - if(!this.isAreaConfigComplete(areaConfig)) { + if(!this.isAreaConfigValid(areaConfig)) { // :TODO: should probably log a warning here return; } @@ -334,4 +718,5 @@ FTNMessageScanTossModule.prototype.record = function(message) { // :TODO: should perhaps record in batches - e.g. start an event, record // to temp location until time is hit or N achieved such that if multiple // messages are being created a .FTN file is not made for each one + */ }; diff --git a/package.json b/package.json index aed5721b..794100a7 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "ptyw.js": "^0.3.7", "sqlite3": "^3.1.1", "ssh2": "^0.4.13", - "string-format": "davidchambers/string-format#mini-language" + "string-format": "davidchambers/string-format#mini-language", + "later" : "1.2.0" }, "engines": { "node": ">=0.12.2" From 1a6af1880161986bc47e697948026e1eb449c6d7 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 28 Feb 2016 22:35:43 -0700 Subject: [PATCH 11/27] Update area scan ID after successful export --- core/scanner_tossers/ftn_bso.js | 78 ++++++++++----------------------- 1 file changed, 24 insertions(+), 54 deletions(-) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 6f45fdb8..87536cb5 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -333,6 +333,16 @@ function FTNMessageScanTossModule() { }); }; + this.setAreaLastScanId = function(areaTag, lastScanId, cb) { + const sql = + `REPLACE INTO message_area_last_scan (scan_toss, area_tag, message_id) + VALUES ("ftn_bso", ?, ?);`; + + msgDb.run(sql, [ areaTag, lastScanId ], err => { + cb(err); + }); + }; + this.getNodeConfigKeyForUplink = function(uplink) { // :TODO: sort by least # of '*' & take top? const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { @@ -619,10 +629,8 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { return cb(new Error('No valid configurations for export')); } - // :TODO: Block exporting (e.g. ignore timer) until export is finished - const getNewUuidsSql = - `SELECT message_uuid + `SELECT message_id, message_uuid FROM message WHERE area_tag = ? AND message_id > ? ORDER BY message_id;`; @@ -649,18 +657,24 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { if(err) { callback(err); } else { - callback(null, rows.map(r => r.message_uuid)); // convert to simple array of UUIDs + if(0 === rows.length) { + let nothingToDoErr = new Error('Nothing to do!'); + nothingToDoErr.noRows = true; + callback(nothingToDoErr); + } else { + callback(null, rows); + } } }); }, - function exportToConfiguredUplinks(msgUuids, callback) { - self.exportMessagesToUplinks(msgUuids, areaConfig, err => { - // :TODO: Log/handle err - callback(null, msgUuids[msgUuids.length - 1]); + function exportToConfiguredUplinks(msgRows, callback) { + const uuidsOnly = msgRows.map(r => r.message_uuid); // conver to array of UUIDs only + self.exportMessagesToUplinks(uuidsOnly, areaConfig, err => { + callback(err, msgRows[msgRows.length - 1].message_id); }); }, function updateLastScanId(newLastScanId, callback) { - callback(null); + self.setAreaLastScanId(areaTag, newLastScanId, callback); } ], function complete(err) { @@ -673,50 +687,6 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { }; FTNMessageScanTossModule.prototype.record = function(message) { - /* - if(!_.has(this, 'moduleConfig.nodes') || - !_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) - { - return; - } - - const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag]; - if(!this.isAreaConfigValid(areaConfig)) { - // :TODO: should probably log a warning here - return; - } - // - // For each uplink, find the best configuration match - // - areaConfig.uplinks.forEach(uplink => { - // :TODO: sort by least # of '*' & take top? - const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { - return Address.fromString(addr).isMatch(uplink); - })[0]; - - if(nodeKey) { - const processOptions = { - nodeConfig : this.moduleConfig.nodes[nodeKey], - network : Config.messageNetworks.ftn.networks[areaConfig.network], - destAddress : Address.fromString(uplink), - networkName : areaConfig.network, - }; - - if(_.isString(processOptions.network.localAddress)) { - // :TODO: move/cache this - e.g. @ startup(). Think about due to Config cache - processOptions.network.localAddress = Address.fromString(processOptions.network.localAddress); - } - - // :TODO: Validate the rest of the matching config -- or do that elsewhere, e.g. startup() - - this.createMessagePacket(message, processOptions); - } - }); - - - // :TODO: should perhaps record in batches - e.g. start an event, record - // to temp location until time is hit or N achieved such that if multiple - // messages are being created a .FTN file is not made for each one - */ + // :TODO: If @immediate, we should do something here! }; From 662d3f232e85c468e49db2fd0044c5ebf6fe100e Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 29 Feb 2016 22:32:51 -0700 Subject: [PATCH 12/27] * Use key name for configured archiver name (e.g. "zip") * Start WIP on mesasge import/toss via schedule * Defaults for message network name --- core/archive_util.js | 2 +- core/config.js | 1 - core/module_util.js | 3 +- core/scanner_tossers/ftn_bso.js | 103 +++++++++++++++++++++++++++++--- 4 files changed, 97 insertions(+), 12 deletions(-) diff --git a/core/archive_util.js b/core/archive_util.js index cbae6d13..f4a3a168 100644 --- a/core/archive_util.js +++ b/core/archive_util.js @@ -75,7 +75,7 @@ module.exports = class ArchiveUtil { } // return first match - const detected = _.find(this.archivers, arch => { + const detected = _.findKey(this.archivers, arch => { const lenNeeded = arch.offset + arch.sig.length; if(buf.length < lenNeeded) { diff --git a/core/config.js b/core/config.js index 47b0003d..c0749e10 100644 --- a/core/config.js +++ b/core/config.js @@ -211,7 +211,6 @@ function getDefaultConfig() { archivers : { zip : { - name : "PKZip", // :TODO: Use key for this sig : "504b0304", offset : 0, compressCmd : "7z", diff --git a/core/module_util.js b/core/module_util.js index fa28d634..931f8194 100644 --- a/core/module_util.js +++ b/core/module_util.js @@ -69,8 +69,7 @@ function loadModulesForCategory(category, iterator, complete) { fs.readdir(Config.paths[category], (err, files) => { if(err) { - iterator(err); - return; + return iterator(err); } const jsModules = files.filter(file => { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 87536cb5..9399b6d0 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -55,8 +55,20 @@ function FTNMessageScanTossModule() { } + this.getDefaultNetworkName = function() { + if(this.moduleConfig.defaultNetwork) { + return this.moduleConfig.defaultNetwork; + } + + const networkNames = Object.keys(Config.messageNetworks.ftn.networks); + if(1 === networkNames.length) { + return networkNames[0]; + } + }; + this.isDefaultDomainZone = function(networkName, address) { - return(networkName === this.moduleConfig.defaultNetwork && address.zone === this.moduleConfig.defaultZone); + const defaultNetworkName = this.getDefaultNetworkName(); + return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone); } this.getOutgoingPacketDir = function(networkName, destAddress) { @@ -288,6 +300,8 @@ function FTNMessageScanTossModule() { return false; } + // :TODO: need to check more! + return true; }; @@ -318,9 +332,6 @@ function FTNMessageScanTossModule() { } }; - this.performImport = function() { - }; - this.getAreaLastScanId = function(areaTag, cb) { const sql = `SELECT area_tag, message_id @@ -576,6 +587,43 @@ function FTNMessageScanTossModule() { ); }, cb); // complete }; + + this.importMessagesFromDirectory = function(importDir, cb) { + async.waterfall( + [ + function getPossibleFiles(callback) { + fs.readdir(importDir, (err, files) => { + callback(err, files); + }); + }, + function identify(files, callback) { + async.map(files, (f, transform) => { + let entry = { file : f }; + + if('.pkt' === paths.extname(f)) { + entry.type = 'packet'; + transform(null, entry); + } else { + const fullPath = paths.join(importDir, f); + self.archUtil.detectType(fullPath, (err, archName) => { + entry.type = archName; + transform(null, entry); + }); + } + }, (err, identifiedFiles) => { + callback(err, identifiedFiles); + }); + }, + function importPacketFiles(identifiedFiles, callback) { + + } + ], + err => { + cb(err); + } + ); + }; + } require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); @@ -592,7 +640,7 @@ FTNMessageScanTossModule.prototype.startup = function(cb) { if(!exporting) { exporting = true; - Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message export...'); + Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message scan/export...'); this.performExport(err => { exporting = false; @@ -605,6 +653,24 @@ FTNMessageScanTossModule.prototype.startup = function(cb) { // :TODO: monitor file for changes/existance with gaze } } + + const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import); + if(importSchedule) { + if(importSchedule.sched) { + let importing = false; + this.importTimer = later.setInterval( () => { + if(!importing) { + importing = true; + + Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message import/toss...'); + + this.performImport(err => { + importing = false; + }); + } + }, importSchedule.sched); + } + } } FTNMessageScanTossModule.super_.prototype.startup.call(this, cb); @@ -620,13 +686,28 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) { FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }; +FTNMessageScanTossModule.prototype.performImport = function(cb) { + if(!this.hasValidConfiguration()) { + return cb(new Error('Missing or invalid configuration')); + } + + var self = this; + + async.each( [ 'inbound', 'secInbound' ], (importDir, nextDir) => { + self.importMessagesFromDirectory(self.moduleConfig.paths[importDir], err => { + + nextDir(); + }); + }, cb); +}; + FTNMessageScanTossModule.prototype.performExport = function(cb) { // // We're only concerned with areas related to FTN. For each area, loop though // and let's find out what messages need exported. // if(!this.hasValidConfiguration()) { - return cb(new Error('No valid configurations for export')); + return cb(new Error('Missing or invalid configuration')); } const getNewUuidsSql = @@ -668,9 +749,15 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { }); }, function exportToConfiguredUplinks(msgRows, callback) { - const uuidsOnly = msgRows.map(r => r.message_uuid); // conver to array of UUIDs only + const uuidsOnly = msgRows.map(r => r.message_uuid); // convert to array of UUIDs only self.exportMessagesToUplinks(uuidsOnly, areaConfig, err => { - callback(err, msgRows[msgRows.length - 1].message_id); + const newLastScanId = msgRows[msgRows.length - 1].message_id; + + Log.info( + { messagesExported : msgRows.length, newLastScanId : newLastScanId }, + 'Export complete'); + + callback(err, newLastScanId); }); }, function updateLastScanId(newLastScanId, callback) { From 5c324788fe7ac54d4237d3ca5f85b39003d72d35 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 1 Mar 2016 22:42:29 -0700 Subject: [PATCH 13/27] * Minor work on FTN/BSO import * Minor work on message network docs --- core/config.js | 2 + core/scanner_tossers/ftn_bso.js | 26 +++++++- docs/msg_networks.md | 105 ++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 docs/msg_networks.md diff --git a/core/config.js b/core/config.js index c0749e10..33a55d09 100644 --- a/core/config.js +++ b/core/config.js @@ -245,6 +245,8 @@ function getDefaultConfig() { outbound : paths.join(__dirname, './../mail/ftn_out/'), inbound : paths.join(__dirname, './../mail/ftn_in/'), secInbound : paths.join(__dirname, './../mail/ftn_secin/'), + + // :TODO: use general temp path - system temp by default...or just always system temp? temp : paths.join(__dirname, './../mail/ftn_temp'), }, diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 9399b6d0..b6dcbc28 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -588,6 +588,10 @@ function FTNMessageScanTossModule() { }, cb); // complete }; + this.importMessagesFromPacketFile = function(path, packetFileName, cb) { + cb(null); + }; + this.importMessagesFromDirectory = function(importDir, cb) { async.waterfall( [ @@ -611,11 +615,27 @@ function FTNMessageScanTossModule() { }); } }, (err, identifiedFiles) => { - callback(err, identifiedFiles); + if(err) { + return callback(err); + } + + const fileGroups = _.partition(identifiedFiles, entry => 'packet' === entry.type); + callback(null, fileGroups[0], fileGroups[1]); }); }, - function importPacketFiles(identifiedFiles, callback) { - + function importPacketFiles(packetFiles, bundleFiles, callback) { + async.each(packetFiles, (packetFile, nextFile) => { + self.importMessagesFromPacketFile(importDir, packetFile.file, err => { + // :TODO: check err! + nextFile(); + }); + }, err => { + // :TODO: Handle err! we should try to keep going though... + callback(null, bundleFiles); + }); + }, + function importBundles(bundleFiles, callback) { + // :TODO: for each bundle, extract to temp location -> process each packet } ], err => { diff --git a/docs/msg_networks.md b/docs/msg_networks.md new file mode 100644 index 00000000..17611f0e --- /dev/null +++ b/docs/msg_networks.md @@ -0,0 +1,105 @@ +# Message Networks +Message networks are configured in `messageNetworks` section of `config.hjson`. Each network type has it's own sub section such as `ftn` for FidoNet Technology Network (FTN) style networks. + +## FidoNet Technology Network (FTN) +FTN networks are configured under the `messageNetworks::ftn` section of `config.hjson`. + +### Networks +The `networks` section contains a sub section for network(s) you wish you join your board with. Each entry's key name can be referenced elsewhere in `config.hjson` for FTN oriented configurations. + +Members: + * `localAddress` (required): FTN address of **your local system** + +Example: +```hjson +{ + networks: { + agoranet: { + localAddress: "46:3/102" + } + } +} +``` + +### Areas +The `areas` section defines a mapping of local **area tags** to a message network (from `networks` described previously), a FTN area tag, and remote uplink address(s). This section can be thought of similar to the *AREAS.BBS* file used by other BBS packages. + +Members: + * `network` (required): Associated network from the `networks` section + * `tag` (required): FTN area tag + * `uplinks`: An array of FTN address uplink(s) for this network + +Example: +```hjson +{ + ftn: { + areas: { + agoranet_bbs: { + network: agoranet + tag: AGN_BBS + uplinks: "46:1/100" + } + } + } +} +``` + +### BSO Import / Export +The scanner/tosser module `ftn_bso` provides **B**inkley **S**tyle **O**utbound (BSO) import/toss & scan/export of messages EchoMail and NetMail messages. Configuration is supplied in `config.hjson` under `scannerTossers::ftn_bso`. + +Members: + * `defaultZone` (required): Sets the default BSO outbound zone + * `defaultNetwork` (optional): Sets the default network name from `messageNetworks::ftn::networks`. **Required if more than one network is defined**. + * `paths` (optional): Override default paths set by the system. This section may contain `outbound`, `inbound`, and `secInbound`. + * `packetTargetByteSize` (optional): Overrides the system *target* packet (.pkt) size of 512000 bytes (512k) + * `bundleTargetByteSize` (optional): Overrides the system *target* ArcMail bundle size of 2048000 bytes (2M) + * `schedule` (required): See Scheduling + * `nodes` (required): See Nodes + +#### Nodes +The `nodes` section defines how to export messages for one or more uplinks. + +A node entry starts with a FTN style address (up to 5D) **as a key** in `config.hjson`. This key may contain wildcard(s) for net/zone/node/point/domain. + +Members: + * `packetType` (optional): `2`, `2.2`, or `2+`. Defaults to `2+` for modern mailer compatiability + * `packetPassword` (optional): Password for the packet + * `encoding` (optional): Encoding to use for message bodies; Defaults to `utf-8` + * `archiveType` (optional): Specifies the archive type for ArcMail bundles. Must be a valid archiver name such as `zip` (See archiver configuration) + +Example: +```hjson +{ + ftn_bso: { + nodes: { + "46:*: { + packetType: 2+ + packetPassword: mypass + encoding: cp437 + archiveType: zip + } + } + } +} +``` + +#### Scheduling +Schedules can be defined for importing and exporting via `import` and `export` under `schedule`. Each entry is allowed a "free form" text and/or special indicators for immediate export or watch file triggers. + + * `@immediate`: Currently only makes sense for exporting: A message will be immediately exported if this trigger is defined in a schedule. + * `@watch:/path/to/file`: This trigger watches the path specified for changes and will trigger an import or export when such events occur. + * Free form text can be things like `at 5:00 pm` or `every 2 hours`. + +See [Later text parsing documentation](http://bunkat.github.io/later/parsers.html#text) for more information. + +Example: +```hjson +{ + ftn_bso: { + schedule: { + import: every 1 hours or @watch:/path/to/watchfile.ext + export: every 1 hours or @immediate + } + } +} +``` \ No newline at end of file From 6094bed07fa0e176d5f110a1d1a6cf26e34ebd78 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 3 Mar 2016 22:54:32 -0700 Subject: [PATCH 14/27] * Use node-temp for temp file creation, cleanup, etc. * Lots of WIP on FTN BSO import * Fix double callbacks in ArchiveUtil * Impl ArchiveUtil.extractTo() * Update bunyan --- core/archive_util.js | 58 +++++++--- core/config.js | 5 +- core/scanner_tossers/ftn_bso.js | 191 +++++++++++++++++++++++++------- package.json | 5 +- 4 files changed, 194 insertions(+), 65 deletions(-) diff --git a/core/archive_util.js b/core/archive_util.js index f4a3a168..914e5e47 100644 --- a/core/archive_util.js +++ b/core/archive_util.js @@ -51,13 +51,17 @@ module.exports = class ArchiveUtil { } } - haveArchiver(archType) { + getArchiver(archType) { if(!archType) { - return false; + return; } archType = archType.toLowerCase(); - return archType in this.archivers; + return this.archivers[archType]; + } + + haveArchiver(archType) { + return this.getArchiver(archType) ? true : false; } detectType(path, cb) { @@ -92,15 +96,13 @@ module.exports = class ArchiveUtil { } compressTo(archType, archivePath, files, cb) { - archType = archType.toLowerCase(); - const archiver = this.archivers[archType]; + const archiver = this.getArchiver(archType); if(!archiver) { - cb(new Error('Unknown archive type: ' + archType)); - return; + return cb(new Error(`Unknown archive type: ${archType}`)); } - let args = _.clone(archiver.compressArgs); // don't much with orig + let args = _.clone(archiver.compressArgs); // don't muck with orig for(let i = 0; i < args.length; ++i) { args[i] = args[i].format({ archivePath : archivePath, @@ -108,18 +110,42 @@ module.exports = class ArchiveUtil { }); } - let comp = pty.spawn(archiver.compressCmd, args, { - cols : 80, - rows : 24, - // :TODO: cwd - }); + let comp = pty.spawn(archiver.compressCmd, args, this.getPtyOpts()); - comp.on('exit', exitCode => { - cb(exitCode ? new Error('Compression failed with exit code: ' + exitCode) : null); + comp.once('exit', exitCode => { + cb(exitCode ? new Error(`Compression failed with exit code: ${exitCode}`) : null); }); } extractTo(archivePath, extractPath, archType, cb) { - + const archiver = this.getArchiver(archType); + + if(!archiver) { + return cb(new Error(`Unknown archive type: ${archType}`)); + } + + let args = _.clone(archiver.decompressArgs); // don't muck with orig + for(let i = 0; i < args.length; ++i) { + args[i] = args[i].format({ + archivePath : archivePath, + extractPath : extractPath, + }); + } + + let comp = pty.spawn(archiver.decompressCmd, args, this.getPtyOpts()); + + comp.once('exit', exitCode => { + cb(exitCode ? new Error(`Decompression failed with exit code: ${exitCode}`) : null); + }); + } + + getPtyOpts() { + return { + // :TODO: cwd + name : 'enigma-archiver', + cols : 80, + rows : 24, + env : process.env, + }; } } diff --git a/core/config.js b/core/config.js index 33a55d09..7525f147 100644 --- a/core/config.js +++ b/core/config.js @@ -216,7 +216,7 @@ function getDefaultConfig() { compressCmd : "7z", compressArgs : [ "a", "-tzip", "{archivePath}", "{fileList}" ], decompressCmd : "7z", - decompressArgs : [ "e", "-o{extractDir}", "{archivePath}" ] + decompressArgs : [ "e", "-o{extractPath}", "{archivePath}" ] } }, @@ -245,9 +245,6 @@ function getDefaultConfig() { outbound : paths.join(__dirname, './../mail/ftn_out/'), inbound : paths.join(__dirname, './../mail/ftn_in/'), secInbound : paths.join(__dirname, './../mail/ftn_secin/'), - - // :TODO: use general temp path - system temp by default...or just always system temp? - temp : paths.join(__dirname, './../mail/ftn_temp'), }, // diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index b6dcbc28..41a87f93 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -19,6 +19,7 @@ let mkdirp = require('mkdirp'); let async = require('async'); let fs = require('fs'); let later = require('later'); +let temp = require('temp').track(); // track() cleans up temp dir/files for us exports.moduleInfo = { name : 'FTN BSO', @@ -69,7 +70,14 @@ function FTNMessageScanTossModule() { this.isDefaultDomainZone = function(networkName, address) { const defaultNetworkName = this.getDefaultNetworkName(); return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone); - } + }; + + this.getNetworkNameByAddress = function(address) { + return _.findKey(Config.messageNetworks.ftn.networks, network => { + const networkAddress = Address.fromString(network.localAddress); + return !_.isUndefined(networkAddress) && address.isEqual(networkAddress); + }); + }; this.getOutgoingPacketDir = function(networkName, destAddress) { let dir = this.moduleConfig.paths.outbound; @@ -306,6 +314,10 @@ function FTNMessageScanTossModule() { }; this.parseScheduleString = function(schedStr) { + if(!schedStr) { + return; // nothing to parse! + } + let schedule = {}; const m = SCHEDULE_REGEXP.exec(schedStr); @@ -413,7 +425,7 @@ function FTNMessageScanTossModule() { packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; // use current message ID for filename seed - const pktFileName = self.getOutgoingPacketFileName(exportOpts.exportDir, message.messageId); + const pktFileName = self.getOutgoingPacketFileName(exportOpts.tempDir, message.messageId); exportedFiles.push(pktFileName); ws = fs.createWriteStream(pktFileName); @@ -475,7 +487,7 @@ function FTNMessageScanTossModule() { packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; // use current message ID for filename seed - const pktFileName = self.getOutgoingPacketFileName(exportOpts.exportDir, remainMessageId); + const pktFileName = self.getOutgoingPacketFileName(exportOpts.tempDir, remainMessageId); exportedFiles.push(pktFileName); ws = fs.createWriteStream(pktFileName); @@ -512,7 +524,6 @@ function FTNMessageScanTossModule() { network : Config.messageNetworks.ftn.networks[areaConfig.network], destAddress : Address.fromString(uplink), networkName : areaConfig.network, - exportDir : self.moduleConfig.paths.temp, }; if(_.isString(exportOpts.network.localAddress)) { @@ -524,7 +535,8 @@ function FTNMessageScanTossModule() { async.waterfall( [ function createTempDir(callback) { - mkdirp(exportOpts.exportDir, err => { + temp.mkdir('enigftnexport--', (err, tempDir) => { + exportOpts.tempDir = tempDir; callback(err); }); }, @@ -551,7 +563,7 @@ function FTNMessageScanTossModule() { } // adjust back to temp path - const tempBundlePath = paths.join(exportOpts.exportDir, paths.basename(bundlePath)); + const tempBundlePath = paths.join(exportOpts.tempDir, paths.basename(bundlePath)); self.archUtil.compressTo( exportOpts.nodeConfig.archiveType, @@ -579,71 +591,164 @@ function FTNMessageScanTossModule() { fs.rename(oldPath, newPath, nextFile); } }, callback); + }, + function cleanUpTempDir(callback) { + temp.cleanup((err, stats) => { + Log.trace( + Object.assign(stats, { tempDir : exportOpts.tempDir }), + 'Temporary directory cleaned up'); + }); } ], err => { - nextUplink(); + nextUplink(); } ); }, cb); // complete }; - this.importMessagesFromPacketFile = function(path, packetFileName, cb) { - cb(null); + this.importMessagesFromPacketFile = function(packetPath, cb) { + const packet = new ftnMailPacket.Packet(); + + // :TODO: packet.read() should have a way to cancel iteration... + let localNetworkName; + packet.read(packetPath, (entryType, entryData) => { + if('header' === entryType) { + // + // Discover if this packet is for one of our network(s) + // + localNetworkName = self.getNetworkNameByAddress(entryData.destAddress); + + } else if(localNetworkName && 'message' === entryType) { + const message = entryData; // so we ref something reasonable :) + const areaTag = message.meta.FtnProperty.ftn_area; + + // :TODO: we need to know if this message is a dupe - UUID will be the same if MSGID, but if not present... what to do? + // :TODO: lookup and set message.areaTag if match + // :TODO: check SEEN-BY for echo + // :TODO: Handle area vs Netmail - Via, etc. + // :TODO: Handle PATH + // :TODO: handle REPLY kludges... set local ID when possible + if(areaTag) { + // + // Find local area tag + // + } + } + }, err => { + cb(err); + }); }; + + this.importPacketFilesFromDirectory = function(importDir, cb) { + async.waterfall( + [ + function getPacketFiles(callback) { + fs.readdir(importDir, (err, files) => { + if(err) { + return callback(err); + } + callback(null, files.filter(f => '.pkt' === paths.extname(f))); + }); + }, + function importPacketFiles(packetFiles, callback) { + let rejects = []; + async.each(packetFiles, (packetFile, nextFile) => { + self.importMessagesFromPacketFile(paths.join(importDir, packetFile), err => { + // :TODO: check err -- log / track rejects, etc. + if(err) { + rejects.push(packetFile); + } + nextFile(); + }); + }, err => { + // :TODO: Handle err! we should try to keep going though... + callback(err, packetFiles, rejects); + }); + }, + function handleProcessedFiles(packetFiles, rejects, callback) { + async.each(packetFiles, (packetFile, nextFile) => { + const fullPath = paths.join(importDir, packetFile); + if(rejects.indexOf(packetFile) > -1) { + // :TODO: rename to .bad, perhaps move to a rejects dir + log + nextFile(); + } else { + //fs.unlink(fullPath, err => { + nextFile(); + //}); + } + }, err => { + callback(err); + }); + } + ], + err => { + cb(err); + } + ); }; this.importMessagesFromDirectory = function(importDir, cb) { async.waterfall( [ - function getPossibleFiles(callback) { - fs.readdir(importDir, (err, files) => { - callback(err, files); + // start with .pkt files + function importPacketFiles(callback) { + self.importPacketFilesFromDirectory(importDir, err => { + callback(err); }); }, - function identify(files, callback) { - async.map(files, (f, transform) => { - let entry = { file : f }; + function discoverBundles(callback) { + fs.readdir(importDir, (err, files) => { + files = files.filter(f => '.pkt' !== paths.extname(f)); - if('.pkt' === paths.extname(f)) { - entry.type = 'packet'; - transform(null, entry); - } else { - const fullPath = paths.join(importDir, f); + async.map(files, (file, transform) => { + const fullPath = paths.join(importDir, file); self.archUtil.detectType(fullPath, (err, archName) => { - entry.type = archName; - transform(null, entry); + transform(null, { path : fullPath, archName : archName } ); }); - } - }, (err, identifiedFiles) => { + }, (err, bundleFiles) => { + callback(err, bundleFiles); + }); + }); + }, + function createTempDir(bundleFiles, callback) { + temp.mkdir('enigftnimport-', (err, tempDir) => { + callback(err, bundleFiles, tempDir); + }); + }, + function importBundles(bundleFiles, tempDir, callback) { + async.each(bundleFiles, (bundleFile, nextFile) => { + if(_.isUndefined(bundleFile.archName)) { + // :TODO: log? + return nextFile(); // unknown archive type + } + + self.archUtil.extractTo( + bundleFile.path, + tempDir, + bundleFile.archName, + err => { + nextFile(); + } + ); + }, err => { if(err) { return callback(err); } - const fileGroups = _.partition(identifiedFiles, entry => 'packet' === entry.type); - callback(null, fileGroups[0], fileGroups[1]); + // + // All extracted - import .pkt's + // + self.importPacketFilesFromDirectory(tempDir, err => { + callback(err); + }); }); - }, - function importPacketFiles(packetFiles, bundleFiles, callback) { - async.each(packetFiles, (packetFile, nextFile) => { - self.importMessagesFromPacketFile(importDir, packetFile.file, err => { - // :TODO: check err! - nextFile(); - }); - }, err => { - // :TODO: Handle err! we should try to keep going though... - callback(null, bundleFiles); - }); - }, - function importBundles(bundleFiles, callback) { - // :TODO: for each bundle, extract to temp location -> process each packet } - ], + ], err => { cb(err); - } + } ); }; - } require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); diff --git a/package.json b/package.json index 794100a7..0d1389ff 100644 --- a/package.json +++ b/package.json @@ -16,10 +16,11 @@ "async": "^1.5.1", "binary": "0.3.x", "buffers": "0.1.x", - "bunyan": "1.5.x", + "bunyan": "^1.7.1", "gaze": "^0.5.2", "hjson": "1.7.x", "iconv-lite": "^0.4.13", + "later": "1.2.0", "lodash": "^3.10.1", "minimist": "1.2.x", "mkdirp": "0.5.x", @@ -29,7 +30,7 @@ "sqlite3": "^3.1.1", "ssh2": "^0.4.13", "string-format": "davidchambers/string-format#mini-language", - "later" : "1.2.0" + "temp": "^0.8.3" }, "engines": { "node": ">=0.12.2" From ad0296addff3cf6b6e488b2a5c3517a220a8624e Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 8 Mar 2016 22:30:04 -0700 Subject: [PATCH 15/27] * Change FTN packet read() to use async iterator * createMessageUuidAlternate(): Mmethod for FTN message v5 UUID generation when no MSGID to work with * parseAbbreviatedNetNodeList() now works properly * Add core/uuid_util.js for various UUID utilities such as v5 named UUID generation * Fix message meta load/retrieval * Add lookup for REPLY kludge -> MSGID -> local reply IDs * Fix SEEN-BY additions @ export * Don't override MSGIDs if they already exist * Store MSGID @ export so it can be inspected later * Add import functionality (working, but WIP!) * Clean up bundles and packets after import --- core/database.js | 2 +- core/ftn_address.js | 2 +- core/ftn_mail_packet.js | 213 +++++++++++---------- core/ftn_util.js | 113 ++++++------ core/message.js | 231 ++++++++++++++++++----- core/scanner_tossers/ftn_bso.js | 315 ++++++++++++++++++++++++-------- core/uuid_util.js | 41 +++++ 7 files changed, 628 insertions(+), 289 deletions(-) create mode 100644 core/uuid_util.js diff --git a/core/database.js b/core/database.js index 3e3b6faf..35e9cc87 100644 --- a/core/database.js +++ b/core/database.js @@ -175,7 +175,7 @@ function createMessageBaseTables() { ' meta_category INTEGER NOT NULL,' + ' meta_name VARCHAR NOT NULL,' + ' meta_value VARCHAR NOT NULL,' + - ' UNIQUE(message_id, meta_category, meta_name, meta_value),' + + ' UNIQUE(message_id, meta_category, meta_name, meta_value),' + // why unique here? ' FOREIGN KEY(message_id) REFERENCES message(message_id)' + ');' ); diff --git a/core/ftn_address.js b/core/ftn_address.js index 880828d9..3e849b55 100644 --- a/core/ftn_address.js +++ b/core/ftn_address.js @@ -107,7 +107,7 @@ module.exports = class Address { } */ - isMatch(pattern) { + isPatternMatch(pattern) { const addr = this.getMatchAddr(pattern); if(addr) { return ( diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index ecc0f1c8..5f4aeac1 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -32,6 +32,7 @@ const FTN_PACKET_HEADER_SIZE = 58; // fixed header size const FTN_PACKET_HEADER_TYPE = 2; const FTN_PACKET_MESSAGE_TYPE = 2; const FTN_PACKET_BAUD_TYPE_2_2 = 2; +const NULL_TERM_BUFFER = new Buffer( [ 0x00 ] ); // SAUCE magic header + version ("00") const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); @@ -171,7 +172,6 @@ exports.PacketHeader = PacketHeader; // http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt // function Packet() { - var self = this; this.parsePacketHeader = function(packetBuffer, cb) { @@ -509,115 +509,105 @@ function Packet() { } ); }; + + this.parsePacketMessages = function(packetBuffer, iterator, cb) { + binary.parse(packetBuffer) + .word16lu('messageType') + .word16lu('ftn_orig_node') + .word16lu('ftn_dest_node') + .word16lu('ftn_orig_network') + .word16lu('ftn_dest_network') + .word16lu('ftn_attr_flags') + .word16lu('ftn_cost') + .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max + .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max + .scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max + .scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max6 + .scan('message', NULL_TERM_BUFFER) + .tap(function tapped(msgData) { // no arrow function; want classic this + if(!msgData.messageType) { + // end marker -- no more messages + return cb(null); + } + + if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) { + return cb(new Error('Unsupported message type: ' + msgData.messageType)); + } + + const read = + 14 + // fixed header size + msgData.modDateTime.length + 1 + + msgData.toUserName.length + 1 + + msgData.fromUserName.length + 1 + + msgData.subject.length + 1 + + msgData.message.length + 1; + + // + // Convert null terminated arrays to strings + // + let convMsgData = {}; + [ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => { + convMsgData[k] = iconv.decode(msgData[k], 'CP437'); + }); - this.parsePacketMessages = function(messagesBuffer, iterator, cb) { - const NULL_TERM_BUFFER = new Buffer( [ 0 ] ); - - var count = 0; + // + // The message body itself is a special beast as it may + // contain an origin line, kludges, SAUCE in the case + // of ANSI files, etc. + // + let msg = new Message( { + toUserName : convMsgData.toUserName, + fromUserName : convMsgData.fromUserName, + subject : convMsgData.subject, + modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime), + }); + + msg.meta.FtnProperty = {}; + msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node; + msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node; + msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; + msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; + msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; + msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; - binary.stream(messagesBuffer).loop(function looper(end, vars) { - // - // Some variable names used here match up directly with well known - // meta data names used with FTN messages. - // - this - .word16lu('messageType') - .word16lu('ftn_orig_node') - .word16lu('ftn_dest_node') - .word16lu('ftn_orig_network') - .word16lu('ftn_dest_network') - .word16lu('ftn_attr_flags') - .word16lu('ftn_cost') - .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max - .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max - .scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max - .scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max - .scan('message', NULL_TERM_BUFFER) - .tap(function tapped(msgData) { - if(!msgData.messageType) { - // end marker -- no more messages - end(); - return; - } - - if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) { - end(); - // :TODO: This is probably a bug if we hit a bad message after at leats one iterate - cb(new Error('Unsupported message type: ' + msgData.messageType)); - return; + self.processMessageBody(msgData.message, messageBodyData => { + msg.message = messageBodyData.message; + msg.meta.FtnKludge = messageBodyData.kludgeLines; + + if(messageBodyData.tearLine) { + msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine; } - ++count; - - // - // Convert null terminated arrays to strings - // - let convMsgData = {}; - [ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => { - convMsgData[k] = iconv.decode(msgData[k], 'CP437'); - }); - - // - // The message body itself is a special beast as it may - // contain special origin lines, kludges, SAUCE in the case - // of ANSI files, etc. - // - let msg = new Message( { - toUserName : convMsgData.toUserName, - fromUserName : convMsgData.fromUserName, - subject : convMsgData.subject, - modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime), - }); - - msg.meta.FtnProperty = {}; - msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node; - msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node; - msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; - msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; - msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; - msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; - - self.processMessageBody(msgData.message, function processed(messageBodyData) { - msg.message = messageBodyData.message; - msg.meta.FtnKludge = messageBodyData.kludgeLines; - - if(messageBodyData.tearLine) { - msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine; - } - if(messageBodyData.seenBy.length > 0) { - msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy; - } - if(messageBodyData.area) { - msg.meta.FtnProperty.ftn_area = messageBodyData.area; - } - if(messageBodyData.originLine) { - msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine; - } - - // - // Update message UUID, if possible, based on MSGID and AREA - // - if(_.isString(msg.meta.FtnKludge.MSGID) && - _.isString(msg.meta.FtnProperty.ftn_area) && - msg.meta.FtnKludge.MSGID.length > 0 && - msg.meta.FtnProperty.ftn_area.length > 0) - { - msg.uuid = ftn.createMessageUuid( - msg.meta.FtnKludge.MSGID, - msg.meta.FtnProperty.area); - } - - iterator('message', msg); + if(messageBodyData.seenBy.length > 0) { + msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy; + } + + if(messageBodyData.area) { + msg.meta.FtnProperty.ftn_area = messageBodyData.area; + } + + if(messageBodyData.originLine) { + msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine; + } + + const nextBuf = packetBuffer.slice(read); + if(nextBuf.length > 0) { + let next = function(e) { + if(e) { + cb(e); + } else { + self.parsePacketMessages(nextBuf, iterator, cb); + } + }; - --count; - if(0 === count) { - cb(null); - } - }) - }); - }); + iterator('message', msg, next); + } else { + cb(null); + } + }); + }); }; - + this.getMessageEntryBuffer = function(message, options) { let basicHeader = new Buffer(34); @@ -664,7 +654,7 @@ function Packet() { }); } } - + // // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 // AREA:CONFERENCE @@ -818,10 +808,15 @@ function Packet() { [ function processHeader(callback) { self.parsePacketHeader(packetBuffer, (err, header) => { - if(!err) { - iterator('header', header); + if(err) { + return callback(err); } - callback(err); + + let next = function(e) { + callback(e); + }; + + iterator('header', header, next); }); }, function processMessages(callback) { @@ -881,7 +876,7 @@ Packet.prototype.read = function(pathOrBuffer, iterator, cb) { }); } ], - function complete(err) { + err => { cb(err); } ); diff --git a/core/ftn_util.js b/core/ftn_util.js index 3a347b2b..595ffca7 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -4,6 +4,7 @@ let Config = require('./config.js').config; let Address = require('./ftn_address.js'); let FNV1a = require('./fnv1a.js'); +let createNamedUUID = require('./uuid_util.js').createNamedUUID; let _ = require('lodash'); let assert = require('assert'); @@ -22,6 +23,7 @@ let packageJson = require('../package.json'); exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; exports.getMessageSerialNumber = getMessageSerialNumber; exports.createMessageUuid = createMessageUuid; +exports.createMessageUuidAlternate = createMessageUuidAlternate; exports.getDateFromFtnDateTime = getDateFromFtnDateTime; exports.getDateTimeString = getDateTimeString; @@ -100,42 +102,43 @@ function getDateTimeString(m) { return m.format('DD MMM YY HH:mm:ss'); } +// +// Create a v5 named UUID given a message ID ("MSGID") and +// FTN area tag ("AREA"). +// +// This is similar to CrashMail +// See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c +// function createMessageUuid(ftnMsgId, ftnArea) { - // - // v5 UUID generation code based on the work here: - // https://github.com/download13/uuidv5/blob/master/uuid.js - // - // Note: CrashMail uses MSGID + AREA, so we go with that as well: - // https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c - // - if(!Buffer.isBuffer(ftnMsgId)) { - ftnMsgId = iconv.encode(ftnMsgId, 'CP437'); - } + assert(_.isString(ftnMsgId)); + assert(_.isString(ftnArea)); - ftnArea = ftnArea || ''; // AREA is optional - if(!Buffer.isBuffer(ftnArea)) { - ftnArea = iconv.encode(ftnArea, 'CP437'); - } + ftnMsgId = iconv.encode(ftnMsgId, 'CP437'); + ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437'); - const ns = new Buffer(ENIGMA_FTN_MSGID_NAMESPACE); + return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] ))); +}; - let digest = createHash('sha1').update( - Buffer.concat([ ns, ftnMsgId, ftnArea ])).digest(); - - let u = new Buffer(16); - - // bbbb - bb - bb - bb - bbbbbb - digest.copy(u, 0, 0, 4); // time_low - digest.copy(u, 4, 4, 6); // time_mid - digest.copy(u, 6, 6, 8); // time_hi_and_version - - u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101) - u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10 - u[9] = digest[9]; +// +// Create a v5 named UUID given a FTN area tag ("AREA"), +// create/modified date, subject, and message body +// +// This method should be used as a backup for when a MSGID is +// not available in which createMessageUuid() above should be +// used instead. +// +function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) { + assert(_.isString(ftnArea)); + assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp)); + assert(_.isString(subject)); + assert(_.isString(msgBody)); + + ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437'); + modTimestamp = iconv.encode(getDateTimeString(modTimestamp), 'CP437'); + subject = iconv.encode(subject.toUpperCase().trim(), 'CP437'); + msgBody = iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437'); - digest.copy(u, 10, 10, 16); - - return uuid.unparse(u); // to string + return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] ))); } function getMessageSerialNumber(messageId) { @@ -274,6 +277,9 @@ function getAbbreviatedNetNodeList(netNodes) { let abbrList = ''; let currNet; netNodes.forEach(netNode => { + if(_.isString(netNode)) { + netNode = Address.fromString(netNode); + } if(currNet !== netNode.net) { abbrList += `${netNode.net}/`; currNet = netNode.net; @@ -284,29 +290,24 @@ function getAbbreviatedNetNodeList(netNodes) { return abbrList.trim(); // remove trailing space } +// +// Parse an abbreviated net/node list commonly used for SEEN-BY and PATH +// function parseAbbreviatedNetNodeList(netNodes) { - // - // Make sure we have an array of objects. - // Allow for a single object or string(s) - // - if(!_.isArray(netNodes)) { - if(_.isString(netNodes)) { - netNodes = netNodes.split(' '); - } else { - netNodes = [ netNodes ]; - } - } - - // - // Convert any strings to parsed address objects - // - return netNodes.map(a => { - if(_.isObject(a)) { - return a; - } else { - return Address.fromString(a); - } - }); + const re = /([0-9]+)\/([0-9]+)\s?|([0-9]+)\s?/g; + let net; + let m; + let results = []; + while(null !== (m = re.exec(netNodes))) { + if(m[1] && m[2]) { + net = parseInt(m[1]); + results.push(new Address( { net : net, node : parseInt(m[2]) } )); + } else if(net) { + results.push(new Address( { net : net, node : parseInt(m[3]) } )); + } + } + + return results; } // @@ -348,8 +349,12 @@ function getUpdatedSeenByEntries(existingEntries, additions) { if(!_.isArray(existingEntries)) { existingEntries = [ existingEntries ]; } + + if(!_.isString(additions)) { + additions = parseAbbreviatedNetNodeList(getAbbreviatedNetNodeList(additions)); + } - additions = parseAbbreviatedNetNodeList(additions).sort(Address.getComparator()); + additions = additions.sort(Address.getComparator()); // // For now, we'll just append a new SEEN-BY entry diff --git a/core/message.js b/core/message.js index 1a5ddd3b..30b72d7c 100644 --- a/core/message.js +++ b/core/message.js @@ -62,22 +62,6 @@ function Message(options) { ts = ts || new Date(); return ts.toISOString(); }; - - /* - Object.defineProperty(this, 'messageId', { - get : function() { - return messageId; - } - }); - - Object.defineProperty(this, 'areaId', { - get : function() { return areaId; }, - set : function(i) { - areaId = i; - } - }); - - */ } Message.WellKnownAreaTags = { @@ -115,9 +99,7 @@ Message.FtnPropertyNames = { FtnDestZone : 'ftn_dest_zone', FtnOrigPoint : 'ftn_orig_point', FtnDestPoint : 'ftn_dest_point', - - - + FtnAttribute : 'ftn_attribute', FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001 @@ -136,6 +118,118 @@ Message.prototype.setLocalFromUserId = function(userId) { this.meta.System.local_from_user_id = userId; }; +Message.getMessageIdByUuid = function(uuid, cb) { + msgDb.get( + `SELECT message_id + FROM message + WHERE message_uuid = ? + LIMIT 1;`, + [ uuid ], + (err, row) => { + if(err) { + cb(err); + } else { + const success = (row && row.message_id); + cb(success ? null : new Error('No match'), success ? row.message_id : null); + } + } + ); +}; + +Message.getMessageIdsByMetaValue = function(category, name, value, cb) { + msgDb.all( + `SELECT message_id + FROM message_meta + WHERE meta_category = ? AND meta_name = ? AND meta_value = ?;`, + [ category, name, value ], + (err, rows) => { + if(err) { + cb(err); + } else { + cb(null, rows.map(r => parseInt(r.message_id))); // return array of ID(s) + } + } + ); +}; + +Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) { + async.waterfall( + [ + function getMessageId(callback) { + Message.getMessageIdByUuid(uuid, (err, messageId) => { + callback(err, messageId); + }); + }, + function getMetaValue(messageId, callback) { + const sql = + `SELECT meta_value + FROM message_meta + WHERE message_id = ? AND message_category = ? AND meta_name = ?;`; + + msgDb.all(sql, [ messageId, category, name ], (err, rows) => { + if(err) { + return callback(err); + } + + if(0 === rows.length) { + return callback(new Error('No value for category/name')); + } + + // single values are returned without an array + if(1 === rows.length) { + return callback(null, rows[0].meta_value); + } + + callback(null, rows.map(r => r.meta_value)); + }); + } + ], + (err, value) => { + cb(err, value); + } + ); +}; + +Message.prototype.loadMeta = function(cb) { + /* + Example of loaded this.meta: + + meta: { + System: { + local_to_user_id: 1234, + }, + FtnProperty: { + ftn_seen_by: [ "1/102 103", "2/42 52 65" ] + } + } + */ + + const sql = + `SELECT meta_category, meta_name, meta_value + FROM message_meta + WHERE message_id = ?;`; + + let self = this; + msgDb.each(sql, [ this.messageId ], (err, row) => { + if(!(row.meta_category in self.meta)) { + self.meta[row.meta_category] = { }; + self.meta[row.meta_category][row.meta_name] = row.meta_value; + } else { + if(!(row.meta_name in self.meta[row.meta_category])) { + self.meta[row.meta_category][row.meta_name] = row.meta_value; + } else { + if(_.isString(self.meta[row.meta_category][row.meta_name])) { + self.meta[row.meta_category][row.meta_name] = [ self.meta[row.meta_category][row.meta_name] ]; + } + + self.meta[row.meta_category][row.meta_name].push(row.meta_value); + } + } + }, err => { + cb(err); + }); +}; + Message.prototype.load = function(options, cb) { assert(_.isString(options.uuid)); @@ -168,8 +262,9 @@ Message.prototype.load = function(options, cb) { ); }, function loadMessageMeta(callback) { - // :TODO: - callback(null); + self.loadMeta(err => { + callback(err); + }); }, function loadHashTags(callback) { // :TODO: @@ -188,27 +283,59 @@ Message.prototype.load = function(options, cb) { ); }; +Message.prototype.persistMetaValue = function(category, name, value, cb) { + const metaStmt = msgDb.prepare( + `INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) + VALUES (?, ?, ?, ?);`); + + if(!_.isArray(value)) { + value = [ value ]; + } + + let self = this; + + async.each(value, (v, next) => { + metaStmt.run(self.messageId, category, name, v, err => { + next(err); + }); + }, err => { + cb(err); + }); +}; + +Message.startTransaction = function(cb) { + msgDb.run('BEGIN;', err => { + cb(err); + }); +}; + +Message.endTransaction = function(hadError, cb) { + msgDb.run(hadError ? 'ROLLBACK;' : 'COMMIT;', err => { + cb(err); + }); +}; + Message.prototype.persist = function(cb) { if(!this.isValid()) { - cb(new Error('Cannot persist invalid message!')); - return; + return cb(new Error('Cannot persist invalid message!')); } - var self = this; - + let self = this; + async.series( [ function beginTransaction(callback) { - msgDb.run('BEGIN;', function transBegin(err) { + Message.startTransaction(err => { callback(err); }); }, function storeMessage(callback) { msgDb.run( - 'INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], - function msgInsert(err) { + `INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, + [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], + function inserted(err) { // use for this scope if(!err) { self.messageId = this.lastID; } @@ -221,26 +348,30 @@ Message.prototype.persist = function(cb) { if(!self.meta) { callback(null); } else { - // :TODO: this should be it's own method such that meta can be updated - var metaStmt = msgDb.prepare( - 'INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) ' + - 'VALUES (?, ?, ?, ?);'); - - for(var metaCategroy in self.meta) { - async.each(Object.keys(self.meta[metaCategroy]), function meta(metaName, next) { - metaStmt.run(self.messageId, Message.MetaCategories[metaCategroy], metaName, self.meta[metaCategroy][metaName], function inserted(err) { - next(err); - }); - }, function complete(err) { - if(!err) { - metaStmt.finalize(function finalized() { - callback(null); - }); - } else { - callback(err); + /* + Example of self.meta: + + meta: { + System: { + local_to_user_id: 1234, + }, + FtnProperty: { + ftn_seen_by: [ "1/102 103", "2/42 52 65" ] } + } + */ + async.each(Object.keys(self.meta), (category, nextCat) => { + async.each(Object.keys(self.meta[category]), (name, nextName) => { + self.persistMetaValue(category, name, self.meta[category][name], err => { + nextName(err); + }); + }, err => { + nextCat(err); }); - } + + }, err => { + callback(err); + }); } }, function storeHashTags(callback) { @@ -248,9 +379,9 @@ Message.prototype.persist = function(cb) { callback(null); } ], - function complete(err) { - msgDb.run(err ? 'ROLLBACK;' : 'COMMIT;', function transEnd(err) { - cb(err, self.messageId); + err => { + Message.endTransaction(err, transErr => { + cb(err ? err : transErr, self.messageId); }); } ); diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 41a87f93..8a83526c 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -20,6 +20,7 @@ let async = require('async'); let fs = require('fs'); let later = require('later'); let temp = require('temp').track(); // track() cleans up temp dir/files for us +let assert = require('assert'); exports.moduleInfo = { name : 'FTN BSO', @@ -72,13 +73,44 @@ function FTNMessageScanTossModule() { return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone); }; - this.getNetworkNameByAddress = function(address) { + this.getNetworkNameByAddress = function(remoteAddress) { return _.findKey(Config.messageNetworks.ftn.networks, network => { - const networkAddress = Address.fromString(network.localAddress); - return !_.isUndefined(networkAddress) && address.isEqual(networkAddress); + const localAddress = Address.fromString(network.localAddress); + return !_.isUndefined(localAddress) && localAddress.isEqual(remoteAddress); }); }; + this.getNetworkNameByAddressPattern = function(remoteAddressPattern) { + return _.findKey(Config.messageNetworks.ftn.networks, network => { + const localAddress = Address.fromString(network.localAddress); + return !_.isUndefined(localAddress) && localAddress.isPatternMatch(remoteAddressPattern); + }); + }; + + this.getLocalAreaTagByFtnAreaTag = function(ftnAreaTag) { + return _.findKey(Config.messageNetworks.ftn.areas, areaConf => { + return areaConf.tag === ftnAreaTag; + }); + }; + + /* + this.getSeenByAddresses = function(messageSeenBy) { + if(!_.isArray(messageSeenBy)) { + messageSeenBy = [ messageSeenBy ]; + } + + let seenByAddrs = []; + messageSeenBy.forEach(sb => { + seenByAddrs = seenByAddrs.concat(ftnUtil.parseAbbreviatedNetNodeList(sb)); + }); + return seenByAddrs; + }; + */ + + this.messageHasValidMSGID = function(msg) { + return _.isString(msg.meta.FtnKludge.MSGID) && msg.meta.FtnKludge.MSGID.length > 0; + }; + this.getOutgoingPacketDir = function(networkName, destAddress) { let dir = this.moduleConfig.paths.outbound; if(!this.isDefaultDomainZone(networkName, destAddress)) { @@ -164,41 +196,6 @@ function FTNMessageScanTossModule() { }); }; - this.exportMessage = function(message, options, cb) { - this.prepareMessage(message, options); - - let packet = new ftnMailPacket.Packet(); - - let packetHeader = new ftnMailPacket.PacketHeader( - options.network.localAddress, - options.destAddress, - options.nodeConfig.packetType); - - packetHeader.password = options.nodeConfig.packetPassword || ''; - - if(message.isPrivate()) { - // :TODO: this should actually be checking for isNetMail()!! - } else { - const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress); - - mkdirp(outgoingDir, err => { - if(err) { - return cb(err); - } - this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => { - console.log(path); - }); - packet.write( - this.getOutgoingPacketFileName(outgoingDir, message), - packetHeader, - [ message ], - { encoding : options.encoding } - ); - }); - } - - }; - this.prepareMessage = function(message, options) { // // Set various FTN kludges/etc. @@ -241,7 +238,8 @@ function FTNMessageScanTossModule() { // When exporting messages, we should create/update SEEN-BY // with remote address(s) we are exporting to. // - const seenByAdditions = [ options.network.localAddress ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks); + const seenByAdditions = + [ `${options.network.localAddress.net}/${options.network.localAddress.node}` ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks); message.meta.FtnProperty.ftn_seen_by = ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions); @@ -256,8 +254,14 @@ function FTNMessageScanTossModule() { // // Additional kludges + // + // Check for existence of MSGID as we may already have stored it from a previous + // export that failed to finish // - message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress); + if(!message.meta.FtnKludge.MSGID) { + message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress); + } + message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); if(!message.meta.FtnKludge.PID) { @@ -369,7 +373,7 @@ function FTNMessageScanTossModule() { this.getNodeConfigKeyForUplink = function(uplink) { // :TODO: sort by least # of '*' & take top? const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { - return Address.fromString(addr).isMatch(uplink); + return Address.fromString(addr).isPatternMatch(uplink); })[0]; return nodeKey; @@ -451,6 +455,19 @@ function FTNMessageScanTossModule() { ws.write(msgBuf); } callback(null); + }, + function updateStoredMeta(callback) { + // + // We want to store some meta as if we had imported + // this message for later reference + // + if(message.meta.FtnKludge.MSGID) { + message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, err => { + callback(err); + }); + } else { + callback(null); + } } ], err => { @@ -607,37 +624,144 @@ function FTNMessageScanTossModule() { }, cb); // complete }; - this.importMessagesFromPacketFile = function(packetPath, cb) { - const packet = new ftnMailPacket.Packet(); + this.setReplyToMsgIdFtnReplyKludge = function(message, cb) { + // + // Given a FTN REPLY kludge, set |message.replyToMsgId|, if possible, + // by looking up an associated MSGID kludge meta. + // + // See also: http://ftsc.org/docs/fts-0009.001 + // + if(!_.isString(message.meta.FtnKludge.REPLY)) { + // nothing to do + return cb(); + } - // :TODO: packet.read() should have a way to cancel iteration... - let localNetworkName; - packet.read(packetPath, (entryType, entryData) => { + Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => { + if(!err) { + assert(1 === msgIds.length); + message.replyToMsgId = msgIds[0]; + } + cb(); + }); + }; + + this.importNetMailToArea = function(localAreaTag, header, message, cb) { + async.series( + [ + function validateDestinationAddress(callback) { + /* + const messageDestAddress = new Address({ + node : message.meta.FtnProperty.ftn_dest_node, + net : message.meta.FtnProperty.ftn_dest_network, + }); + */ + + const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`; + + const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern); + + callback(_.isString(localNetworkName) ? null : new Error('Packet destination is not us')); + }, + function basicSetup(callback) { + message.areaTag = localAreaTag; + + // + // If duplicates are NOT allowed in the area (the default), we need to update + // the message UUID using data available to us. Duplicate UUIDs are internally + // not allowed in our local database. + // + if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) { + if(self.messageHasValidMSGID(message)) { + // Update UUID with our preferred generation method + message.uuid = ftnUtil.createMessageUuid( + message.meta.FtnKludge.MSGID, + message.meta.FtnProperty.ftn_area); + } else { + // Update UUID with alternate/backup generation method + message.uuid = ftnUtil.createMessageUuidAlternate( + message.meta.FtnProperty.ftn_area, + message.modTimestamp, + message.subject, + message.message); + } + } + + callback(null); + }, + function setReplyToMessageId(callback) { + self.setReplyToMsgIdFtnReplyKludge(message, () => { + callback(null); + }); + }, + function persistImport(callback) { + message.persist(err => { + callback(err); + }); + } + ], err => { + cb(err); + } + ); + }; + + // + // Ref. implementations on import: + // * https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c + // https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c + // + this.importMessagesFromPacketFile = function(packetPath, cb) { + let packetHeader; + + new ftnMailPacket.Packet().read(packetPath, (entryType, entryData, next) => { if('header' === entryType) { - // - // Discover if this packet is for one of our network(s) - // - localNetworkName = self.getNetworkNameByAddress(entryData.destAddress); - - } else if(localNetworkName && 'message' === entryType) { - const message = entryData; // so we ref something reasonable :) + packetHeader = entryData; + + const localNetworkName = self.getNetworkNameByAddress(packetHeader.destAddress); + if(!_.isString(localNetworkName)) { + next(new Error('No configuration for this packet')); + } else { + next(null); + } + + } else if('message' === entryType) { + const message = entryData; const areaTag = message.meta.FtnProperty.ftn_area; - // :TODO: we need to know if this message is a dupe - UUID will be the same if MSGID, but if not present... what to do? - // :TODO: lookup and set message.areaTag if match - // :TODO: check SEEN-BY for echo - // :TODO: Handle area vs Netmail - Via, etc. - // :TODO: Handle PATH - // :TODO: handle REPLY kludges... set local ID when possible if(areaTag) { // - // Find local area tag + // EchoMail + // + const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag); + if(localAreaTag) { + self.importNetMailToArea(localAreaTag, packetHeader, message, err => { + if(err) { + if('SQLITE_CONSTRAINT' === err.code) { + Log.info( + { subject : message.subject, uuid : message.uuid }, + 'Not importing non-unique message'); + + return next(null); + } + } + + next(err); + }); + } else { + // + // No local area configured for this import + // + // :TODO: Handle the "catch all" case, if configured + } + } else { + // + // NetMail // } } }, err => { cb(err); - }); }; + }); + }; this.importPacketFilesFromDirectory = function(importDir, cb) { async.waterfall( @@ -672,9 +796,9 @@ function FTNMessageScanTossModule() { // :TODO: rename to .bad, perhaps move to a rejects dir + log nextFile(); } else { - //fs.unlink(fullPath, err => { + fs.unlink(fullPath, err => { nextFile(); - //}); + }); } }, err => { callback(err); @@ -687,7 +811,9 @@ function FTNMessageScanTossModule() { ); }; - this.importMessagesFromDirectory = function(importDir, cb) { + this.importMessagesFromDirectory = function(inboundType, importDir, cb) { + let tempDirectory; + async.waterfall( [ // start with .pkt files @@ -712,21 +838,37 @@ function FTNMessageScanTossModule() { }, function createTempDir(bundleFiles, callback) { temp.mkdir('enigftnimport-', (err, tempDir) => { - callback(err, bundleFiles, tempDir); + tempDirectory = tempDir; + callback(err, bundleFiles); }); }, - function importBundles(bundleFiles, tempDir, callback) { + function importBundles(bundleFiles, callback) { + let rejects = []; + async.each(bundleFiles, (bundleFile, nextFile) => { if(_.isUndefined(bundleFile.archName)) { - // :TODO: log? + Log.info( + { fileName : bundleFile.path }, + 'Unknown bundle archive type'); + + rejects.push(bundleFile.path); + return nextFile(); // unknown archive type } self.archUtil.extractTo( bundleFile.path, - tempDir, + tempDirectory, bundleFile.archName, err => { + if(err) { + Log.info( + { fileName : bundleFile.path, error : err.toString() }, + 'Failed to extract bundle'); + + rejects.push(bundleFile.path); + } + nextFile(); } ); @@ -738,14 +880,39 @@ function FTNMessageScanTossModule() { // // All extracted - import .pkt's // - self.importPacketFilesFromDirectory(tempDir, err => { - callback(err); + self.importPacketFilesFromDirectory(tempDirectory, err => { + callback(null, bundleFiles, rejects); }); }); + }, + function handleProcessedBundleFiles(bundleFiles, rejects, callback) { + async.each(bundleFiles, (bundleFile, nextFile) => { + if(rejects.indexOf(bundleFile.path) > -1) { + // :TODO: rename to .bad, perhaps move to a rejects dir + log + nextFile(); + } else { + fs.unlink(bundleFile.path, err => { + nextFile(); + }); + } + }, err => { + callback(err); + }); } ], err => { - cb(err); + if(tempDirectory) { + temp.cleanup( (errIgnored, stats) => { + Log.trace( + Object.assign(stats, { tempDir : tempDirectory } ), + 'Temporary directory cleaned up' + ); + + cb(err); // orig err + }); + } else { + cb(err); + } } ); }; @@ -818,8 +985,8 @@ FTNMessageScanTossModule.prototype.performImport = function(cb) { var self = this; - async.each( [ 'inbound', 'secInbound' ], (importDir, nextDir) => { - self.importMessagesFromDirectory(self.moduleConfig.paths[importDir], err => { + async.each( [ 'inbound', 'secInbound' ], (inboundType, nextDir) => { + self.importMessagesFromDirectory(inboundType, self.moduleConfig.paths[inboundType], err => { nextDir(); }); @@ -879,7 +1046,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { const newLastScanId = msgRows[msgRows.length - 1].message_id; Log.info( - { messagesExported : msgRows.length, newLastScanId : newLastScanId }, + { areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId }, 'Export complete'); callback(err, newLastScanId); diff --git a/core/uuid_util.js b/core/uuid_util.js new file mode 100644 index 00000000..00e8840c --- /dev/null +++ b/core/uuid_util.js @@ -0,0 +1,41 @@ +/* jslint node: true */ +'use strict'; + +let uuid = require('node-uuid'); +let assert = require('assert'); +let _ = require('lodash'); +let createHash = require('crypto').createHash; + +exports.createNamedUUID = createNamedUUID; + +function createNamedUUID(namespaceUuid, key) { + // + // v5 UUID generation code based on the work here: + // https://github.com/download13/uuidv5/blob/master/uuid.js + // + if(!Buffer.isBuffer(namespaceUuid)) { + namespaceUuid = new Buffer(namespaceUuid); + } + + if(!Buffer.isBuffer(key)) { + key = new Buffer(key); + } + + let digest = createHash('sha1').update( + Buffer.concat( [ namespaceUuid, key ] )).digest(); + + let u = new Buffer(16); + + // bbbb - bb - bb - bb - bbbbbb + digest.copy(u, 0, 0, 4); // time_low + digest.copy(u, 4, 4, 6); // time_mid + digest.copy(u, 6, 6, 8); // time_hi_and_version + + u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101) + u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10 + u[9] = digest[9]; + + digest.copy(u, 10, 10, 16); + + return u; +} \ No newline at end of file From 31ca7d3eaf4cea9231ed77b29c8232652493566f Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 9 Mar 2016 22:32:00 -0700 Subject: [PATCH 16/27] * Don't export imported messages * Some basic code cleanup --- core/ftn_mail_packet.js | 15 +++++------ core/ftn_util.js | 47 ++++++++++----------------------- core/scanner_tossers/ftn_bso.js | 41 +++++++++++++++++----------- 3 files changed, 45 insertions(+), 58 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 5f4aeac1..c8312eeb 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -11,10 +11,8 @@ let _ = require('lodash'); let assert = require('assert'); let binary = require('binary'); let fs = require('fs'); -let util = require('util'); let async = require('async'); let iconv = require('iconv-lite'); -let buffers = require('buffers'); let moment = require('moment'); exports.Packet = Packet; @@ -835,8 +833,11 @@ function Packet() { // Message attributes defined in FTS-0001.016 // http://ftsc.org/docs/fts-0001.016 // +// See also: +// * http://www.skepticfiles.org/aj/basics03.htm +// Packet.Attribute = { - Private : 0x0001, + Private : 0x0001, // Private message / NetMail Crash : 0x0002, Received : 0x0004, Sent : 0x0008, @@ -844,7 +845,7 @@ Packet.Attribute = { InTransit : 0x0020, Orphan : 0x0040, KillSent : 0x0080, - Local : 0x0100, + Local : 0x0100, // Message is from *this* system Hold : 0x0200, Reserved0 : 0x0400, FileRequest : 0x0800, @@ -886,10 +887,6 @@ Packet.prototype.writeHeader = function(ws, packetHeader) { return this.writePacketHeader(packetHeader, ws); }; -Packet.prototype.writeMessage = function(ws, message, options) { - -} - Packet.prototype.writeMessageEntry = function(ws, msgEntry) { ws.write(msgEntry); return msgEntry.length; @@ -918,7 +915,7 @@ Packet.prototype.writeStream = function(ws, messages, options) { if(true === options.terminatePacket) { ws.write(new Buffer( [ 0 ] )); // final extra null term } -} +}; Packet.prototype.write = function(path, packetHeader, messages, options) { if(!_.isArray(messages)) { diff --git a/core/ftn_util.js b/core/ftn_util.js index 595ffca7..fdc5e5a6 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -8,12 +8,8 @@ let createNamedUUID = require('./uuid_util.js').createNamedUUID; let _ = require('lodash'); let assert = require('assert'); -let binary = require('binary'); -let fs = require('fs'); -let util = require('util'); let iconv = require('iconv-lite'); let moment = require('moment'); -let createHash = require('crypto').createHash; let uuid = require('node-uuid'); let os = require('os'); @@ -49,9 +45,6 @@ exports.getQuotePrefix = getQuotePrefix; // const ENIGMA_FTN_MSGID_NAMESPACE = uuid.parse('a5c7ae11-420c-4469-a116-0e9a6d8d2654'); -// Up to 5D FTN address RegExp -const ENIGMA_FTN_ADDRESS_REGEXP = /^([0-9]+:)?([0-9]+)(\/[0-9]+)?(\.[0-9]+)?(@[a-z0-9\-\.]+)?$/i; - // See list here: https://github.com/Mithgol/node-fidonet-jam function stringToNullPaddedBuffer(s, bufLen) { @@ -145,8 +138,6 @@ function getMessageSerialNumber(messageId) { const msSinceEnigmaEpoc = (Date.now() - Date.UTC(2016, 1, 1)); const hash = Math.abs(new FNV1a(msSinceEnigmaEpoc + messageId).value).toString(16); return `00000000${hash}`.substr(-8); - // return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + - // message.messageId)).toString(16)).substr(-8); } // @@ -207,16 +198,6 @@ function getProductIdentifier() { return `ENiGMA1/2 ${version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; } -// -// Return a FSC-0030.001 compliant (http://ftsc.org/docs/fsc-0030.001) MESSAGE-ID -// -// -// -// :TODO: not implemented to spec at all yet :) -function getFTNMessageID(messageId, areaId) { - return messageId + '.' + areaId + '@' + getFTNAddress() + ' ' + getMessageSerialNumber(messageId) -} - // // Return a FRL-1004 style time zone offset for a // 'TZUTC' kludge line @@ -294,20 +275,20 @@ function getAbbreviatedNetNodeList(netNodes) { // Parse an abbreviated net/node list commonly used for SEEN-BY and PATH // function parseAbbreviatedNetNodeList(netNodes) { - const re = /([0-9]+)\/([0-9]+)\s?|([0-9]+)\s?/g; - let net; - let m; - let results = []; - while(null !== (m = re.exec(netNodes))) { - if(m[1] && m[2]) { - net = parseInt(m[1]); - results.push(new Address( { net : net, node : parseInt(m[2]) } )); - } else if(net) { - results.push(new Address( { net : net, node : parseInt(m[3]) } )); - } - } - - return results; + const re = /([0-9]+)\/([0-9]+)\s?|([0-9]+)\s?/g; + let net; + let m; + let results = []; + while(null !== (m = re.exec(netNodes))) { + if(m[1] && m[2]) { + net = parseInt(m[1]); + results.push(new Address( { net : net, node : parseInt(m[2]) } )); + } else if(net) { + results.push(new Address( { net : net, node : parseInt(m[3]) } )); + } + } + + return results; } // diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 8a83526c..d4118e9a 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -184,7 +184,7 @@ function FTNMessageScanTossModule() { let fileName = `${basename}.${moment().format('dd').toLowerCase()}`; async.detectSeries(EXT_SUFFIXES, (suffix, callback) => { const checkFileName = fileName + suffix; - fs.stat(paths.join(basePath, checkFileName), (err, stats) => { + fs.stat(paths.join(basePath, checkFileName), err => { callback((err && 'ENOENT' === err.code) ? true : false); }); }, finalSuffix => { @@ -213,7 +213,8 @@ function FTNMessageScanTossModule() { message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); // :TODO: Need an explicit isNetMail() check - let ftnAttribute = 0; + let ftnAttribute = + ftnMailPacket.Packet.Attribute.Local; // message from our system if(message.isPrivate()) { ftnAttribute |= ftnMailPacket.Packet.Attribute.Private; @@ -230,7 +231,7 @@ function FTNMessageScanTossModule() { } else { // // EchoMail requires some additional properties & kludges - // + // message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(options.network.localAddress); message.meta.FtnProperty.ftn_area = Config.messageNetworks.ftn.areas[message.areaTag].tag; @@ -586,6 +587,10 @@ function FTNMessageScanTossModule() { exportOpts.nodeConfig.archiveType, tempBundlePath, exportedFileNames, err => { + if(err) { + return callback(err); + } + // :TODO: we need to delete the original input file(s) fs.rename(tempBundlePath, bundlePath, err => { callback(err, [ bundlePath ] ); @@ -614,10 +619,13 @@ function FTNMessageScanTossModule() { Log.trace( Object.assign(stats, { tempDir : exportOpts.tempDir }), 'Temporary directory cleaned up'); + + callback(null); }); } ], err => { + // :TODO: do something with |err| ? nextUplink(); } ); @@ -648,16 +656,8 @@ function FTNMessageScanTossModule() { this.importNetMailToArea = function(localAreaTag, header, message, cb) { async.series( [ - function validateDestinationAddress(callback) { - /* - const messageDestAddress = new Address({ - node : message.meta.FtnProperty.ftn_dest_node, - net : message.meta.FtnProperty.ftn_dest_network, - }); - */ - - const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`; - + function validateDestinationAddress(callback) { + const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`; const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern); callback(_.isString(localNetworkName) ? null : new Error('Packet destination is not us')); @@ -698,7 +698,8 @@ function FTNMessageScanTossModule() { callback(err); }); } - ], err => { + ], + err => { cb(err); } ); @@ -1002,10 +1003,18 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { return cb(new Error('Missing or invalid configuration')); } + // + // Select all messages that have a message_id > our last scan ID. + // Additionally exclude messages that have a ftn_attr_flags FtnProperty meta + // as those came via import! + // const getNewUuidsSql = `SELECT message_id, message_uuid - FROM message - WHERE area_tag = ? AND message_id > ? + FROM message m + WHERE area_tag = ? AND message_id > ? AND + (SELECT COUNT(message_id) + FROM message_meta + WHERE message_id = m.message_id AND meta_category = 'FtnProperty' AND meta_name = 'ftn_attr_flags') = 0 ORDER BY message_id;`; var self = this; From 86c659849ca375ae242f42cd3731442b3b829322 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 12 Mar 2016 00:22:06 -0700 Subject: [PATCH 17/27] * Flow file creation for exported bundles based on node configuration - 'crash' is currently the default --- core/scanner_tossers/ftn_bso.js | 78 ++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index d4118e9a..ce37ec46 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -93,6 +93,10 @@ function FTNMessageScanTossModule() { }); }; + this.getExportType = function(nodeConfig) { + return _.isString(nodeConfig.exportType) ? nodeConfig.exportType.toLowerCase() : 'crash'; + }; + /* this.getSeenByAddresses = function(messageSeenBy) { if(!_.isArray(messageSeenBy)) { @@ -143,7 +147,18 @@ function FTNMessageScanTossModule() { return paths.join(basePath, `${name}.${ext}`); }; - this.getOutgoingFlowFileName = function(basePath, destAddress, exportType, extSuffix) { + this.getOutgoingFlowFileName = function(basePath, destAddress, flowType, exportType) { + let basename; + let ext; + + switch(flowType) { + case 'netmail' : ext = `${exportType.toLowerCase()[0]}ut`; break; + case 'ref' : ext = `${exportType.toLowerCase()[0]}lo`; break; + case 'busy' : ext = 'bsy'; break; + case 'request' : ext = 'req'; break; + case 'requests' : ext = 'hrq'; break; + } + if(destAddress.point) { } else { @@ -151,8 +166,22 @@ function FTNMessageScanTossModule() { // Use |destAddress| nnnnNNNN.??? where nnnn is dest net and NNNN is dest // node. This seems to match what Mystic does // - return `${Math.abs(destAddress.net)}${Math.abs(destAddress.node)}.${exportType[1]}${extSuffix}`; + basename = + `0000${destAddress.net.toString(16)}`.substr(-4) + + `0000${destAddress.node.toString(16)}`.substr(-4); } + + return paths.join(basePath, `${basename}.${ext}`); + }; + + this.flowFileAppendRefs = function(filePath, fileRefs, directive, cb) { + const appendLines = fileRefs.reduce( (content, ref) => { + return content + `${directive}${ref}\n`; + }, ''); + + fs.appendFile(filePath, appendLines, err => { + cb(err); + }); }; this.getOutgoingBundleFileName = function(basePath, sourceAddress, destAddress, cb) { @@ -166,7 +195,7 @@ function FTNMessageScanTossModule() { // // Extension is dd? where dd is Su...Mo and ? is 0...Z as collisions arise // - var basename; + let basename; if(destAddress.point) { const pointHex = `000${destAddress.point}`.substr(-3); basename = `0000p${pointHex}`; @@ -229,6 +258,15 @@ function FTNMessageScanTossModule() { message.meta.FtnKludge.Via = message.meta.FtnKludge.Via || []; message.meta.FtnKludge.Via.push(ftnUtil.getVia(options.network.localAddress)); } else { + // + // Set appropriate attribute flag for export type + // + switch(this.getExportType(options.nodeConfig)) { + case 'crash' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Crash; break; + case 'hold' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Hold; break; + // :TODO: Others? + } + // // EchoMail requires some additional properties & kludges // @@ -587,14 +625,7 @@ function FTNMessageScanTossModule() { exportOpts.nodeConfig.archiveType, tempBundlePath, exportedFileNames, err => { - if(err) { - return callback(err); - } - - // :TODO: we need to delete the original input file(s) - fs.rename(tempBundlePath, bundlePath, err => { - callback(err, [ bundlePath ] ); - }); + callback(err, [ tempBundlePath ] ); } ); }); @@ -610,7 +641,30 @@ function FTNMessageScanTossModule() { fs.rename(oldPath, newPath, nextFile); } else { const newPath = paths.join(outgoingDir, paths.basename(oldPath)); - fs.rename(oldPath, newPath, nextFile); + //fs.rename(oldPath, newPath, nextFile); + fs.rename(oldPath, newPath, err => { + if(err) { + // :TODO: Log this - but move on to the next file + return nextFile(); + } + + // + // For bundles, we need to append to the appropriate flow file + // + const flowFilePath = self.getOutgoingFlowFileName( + outgoingDir, + exportOpts.destAddress, + 'ref', + self.getExportType(exportOpts.nodeConfig)); + + // directive of '^' = delete file after transfer + self.flowFileAppendRefs(flowFilePath, [ newPath ], '^', err => { + if(err) { + // :TODO: Log this! + } + nextFile(); + }); + }); } }, callback); }, From a787a2eab399b0aa58213000042ec4a01a2f45dc Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 13 Mar 2016 11:11:51 -0600 Subject: [PATCH 18/27] * Fix collsion with import/export temporary dirs; better use of temp dirs all around * Raw (non-bundle) packet exports are now BSO named (e.g. .cut for crash) --- core/scanner_tossers/ftn_bso.js | 218 ++++++++++++++++++-------------- 1 file changed, 122 insertions(+), 96 deletions(-) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index ce37ec46..b112d474 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -146,19 +146,25 @@ function FTNMessageScanTossModule() { const ext = (true === isTemp) ? 'pk_' : 'pkt'; return paths.join(basePath, `${name}.${ext}`); }; - - this.getOutgoingFlowFileName = function(basePath, destAddress, flowType, exportType) { - let basename; + + this.getOutgoingFlowFileExtension = function(destAddress, flowType, exportType) { let ext; switch(flowType) { - case 'netmail' : ext = `${exportType.toLowerCase()[0]}ut`; break; + case 'mail' : ext = `${exportType.toLowerCase()[0]}ut`; break; case 'ref' : ext = `${exportType.toLowerCase()[0]}lo`; break; case 'busy' : ext = 'bsy'; break; case 'request' : ext = 'req'; break; case 'requests' : ext = 'hrq'; break; } + return ext; + }; + + this.getOutgoingFlowFileName = function(basePath, destAddress, flowType, exportType) { + let basename; + const ext = self.getOutgoingFlowFileExtension(destAddress, flowType, exportType); + if(destAddress.point) { } else { @@ -431,6 +437,7 @@ function FTNMessageScanTossModule() { let ws; let remainMessageBuf; let remainMessageId; + const createTempPacket = !_.isString(exportOpts.nodeConfig.archiveType) || 0 === exportOpts.nodeConfig.archiveType.length; async.each(messageUuids, (msgUuid, nextUuid) => { let message = new Message(); @@ -468,7 +475,7 @@ function FTNMessageScanTossModule() { packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; // use current message ID for filename seed - const pktFileName = self.getOutgoingPacketFileName(exportOpts.tempDir, message.messageId); + const pktFileName = self.getOutgoingPacketFileName(self.exportTempDir, message.messageId, createTempPacket); exportedFiles.push(pktFileName); ws = fs.createWriteStream(pktFileName); @@ -543,7 +550,7 @@ function FTNMessageScanTossModule() { packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; // use current message ID for filename seed - const pktFileName = self.getOutgoingPacketFileName(exportOpts.tempDir, remainMessageId); + const pktFileName = self.getOutgoingPacketFileName(self.exportTempDir, remainMessageId, createTempPacket); exportedFiles.push(pktFileName); ws = fs.createWriteStream(pktFileName); @@ -586,16 +593,11 @@ function FTNMessageScanTossModule() { exportOpts.network.localAddress = Address.fromString(exportOpts.network.localAddress); } - const outgoingDir = self.getOutgoingPacketDir(exportOpts.networkName, exportOpts.destAddress); + const outgoingDir = self.getOutgoingPacketDir(exportOpts.networkName, exportOpts.destAddress); + const exportType = self.getExportType(exportOpts.nodeConfig); async.waterfall( [ - function createTempDir(callback) { - temp.mkdir('enigftnexport--', (err, tempDir) => { - exportOpts.tempDir = tempDir; - callback(err); - }); - }, function createOutgoingDir(callback) { mkdirp(outgoingDir, err => { callback(err); @@ -619,7 +621,7 @@ function FTNMessageScanTossModule() { } // adjust back to temp path - const tempBundlePath = paths.join(exportOpts.tempDir, paths.basename(bundlePath)); + const tempBundlePath = paths.join(self.exportTempDir, paths.basename(bundlePath)); self.archUtil.compressTo( exportOpts.nodeConfig.archiveType, @@ -637,14 +639,29 @@ function FTNMessageScanTossModule() { async.each(exportedFileNames, (oldPath, nextFile) => { const ext = paths.extname(oldPath); if('.pk_' === ext) { - const newPath = paths.join(outgoingDir, paths.basename(oldPath, ext) + '.pkt'); + // + // For a given temporary .pk_ file, we need to move it to the outoing + // directory with the appropriate BSO style filename. + // + const ext = self.getOutgoingFlowFileExtension( + exportOpts.destAddress, + 'mail', + exportType); + + const newPath = paths.join( + outgoingDir, + `${paths.basename(oldPath, 'pk_')}${ext}`); + fs.rename(oldPath, newPath, nextFile); } else { const newPath = paths.join(outgoingDir, paths.basename(oldPath)); - //fs.rename(oldPath, newPath, nextFile); fs.rename(oldPath, newPath, err => { if(err) { // :TODO: Log this - but move on to the next file + Log.warn( + { oldPath : oldPath, newPath : newPath }, + 'Failed moving temporary bundle file!'); + return nextFile(); } @@ -655,27 +672,18 @@ function FTNMessageScanTossModule() { outgoingDir, exportOpts.destAddress, 'ref', - self.getExportType(exportOpts.nodeConfig)); + exportType); // directive of '^' = delete file after transfer self.flowFileAppendRefs(flowFilePath, [ newPath ], '^', err => { if(err) { - // :TODO: Log this! + Log.warn( { path : flowFilePath }, 'Failed appending flow reference record!'); } nextFile(); }); }); } }, callback); - }, - function cleanUpTempDir(callback) { - temp.cleanup((err, stats) => { - Log.trace( - Object.assign(stats, { tempDir : exportOpts.tempDir }), - 'Temporary directory cleaned up'); - - callback(null); - }); } ], err => { @@ -764,7 +772,7 @@ function FTNMessageScanTossModule() { // * https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c // https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c // - this.importMessagesFromPacketFile = function(packetPath, cb) { + this.importMessagesFromPacketFile = function(packetPath, password, cb) { let packetHeader; new ftnMailPacket.Packet().read(packetPath, (entryType, entryData, next) => { @@ -818,7 +826,7 @@ function FTNMessageScanTossModule() { }); }; - this.importPacketFilesFromDirectory = function(importDir, cb) { + this.importPacketFilesFromDirectory = function(importDir, password, cb) { async.waterfall( [ function getPacketFiles(callback) { @@ -832,7 +840,7 @@ function FTNMessageScanTossModule() { function importPacketFiles(packetFiles, callback) { let rejects = []; async.each(packetFiles, (packetFile, nextFile) => { - self.importMessagesFromPacketFile(paths.join(importDir, packetFile), err => { + self.importMessagesFromPacketFile(paths.join(importDir, packetFile), '', err => { // :TODO: check err -- log / track rejects, etc. if(err) { rejects.push(packetFile); @@ -867,13 +875,11 @@ function FTNMessageScanTossModule() { }; this.importMessagesFromDirectory = function(inboundType, importDir, cb) { - let tempDirectory; - async.waterfall( [ // start with .pkt files function importPacketFiles(callback) { - self.importPacketFilesFromDirectory(importDir, err => { + self.importPacketFilesFromDirectory(importDir, '', err => { callback(err); }); }, @@ -891,18 +897,12 @@ function FTNMessageScanTossModule() { }); }); }, - function createTempDir(bundleFiles, callback) { - temp.mkdir('enigftnimport-', (err, tempDir) => { - tempDirectory = tempDir; - callback(err, bundleFiles); - }); - }, function importBundles(bundleFiles, callback) { let rejects = []; async.each(bundleFiles, (bundleFile, nextFile) => { if(_.isUndefined(bundleFile.archName)) { - Log.info( + Log.warn( { fileName : bundleFile.path }, 'Unknown bundle archive type'); @@ -913,11 +913,11 @@ function FTNMessageScanTossModule() { self.archUtil.extractTo( bundleFile.path, - tempDirectory, + self.importTempDir, bundleFile.archName, err => { if(err) { - Log.info( + Log.warn( { fileName : bundleFile.path, error : err.toString() }, 'Failed to extract bundle'); @@ -935,7 +935,8 @@ function FTNMessageScanTossModule() { // // All extracted - import .pkt's // - self.importPacketFilesFromDirectory(tempDirectory, err => { + self.importPacketFilesFromDirectory(self.importTempDir, '', err => { + // :TODO: handle |err| callback(null, bundleFiles, rejects); }); }); @@ -956,71 +957,83 @@ function FTNMessageScanTossModule() { } ], err => { - if(tempDirectory) { - temp.cleanup( (errIgnored, stats) => { - Log.trace( - Object.assign(stats, { tempDir : tempDirectory } ), - 'Temporary directory cleaned up' - ); - - cb(err); // orig err - }); - } else { - cb(err); - } + cb(err); } ); }; + + this.createTempDirectories = function(cb) { + temp.mkdir('enigftnexport-', (err, tempDir) => { + if(err) { + return cb(err); + } + + self.exportTempDir = tempDir; + + temp.mkdir('enigftnimport-', (err, tempDir) => { + self.importTempDir = tempDir; + + cb(err); + }); + }); + }; } require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); FTNMessageScanTossModule.prototype.startup = function(cb) { - Log.info('FidoNet Scanner/Tosser starting up'); + Log.info(`${exports.moduleInfo.name} Scanner/Tosser starting up`); - if(_.isObject(this.moduleConfig.schedule)) { - const exportSchedule = this.parseScheduleString(this.moduleConfig.schedule.export); - if(exportSchedule) { - if(exportSchedule.sched) { - let exporting = false; - this.exportTimer = later.setInterval( () => { - if(!exporting) { - exporting = true; - - Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message scan/export...'); - - this.performExport(err => { - exporting = false; - }); - } - }, exportSchedule.sched); + this.createTempDirectories(err => { + if(err) { + Log.warn( { error : err.toStrong() }, 'Failed creating temporary directories!'); + return cb(err); + } + + if(_.isObject(this.moduleConfig.schedule)) { + const exportSchedule = this.parseScheduleString(this.moduleConfig.schedule.export); + if(exportSchedule) { + if(exportSchedule.sched) { + let exporting = false; + this.exportTimer = later.setInterval( () => { + if(!exporting) { + exporting = true; + + Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message scan/export...'); + + this.performExport( () => { + exporting = false; + }); + } + }, exportSchedule.sched); + } + + if(exportSchedule.watchFile) { + // :TODO: monitor file for changes/existance with gaze + } } - if(exportSchedule.watchFile) { - // :TODO: monitor file for changes/existance with gaze + const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import); + if(importSchedule) { + if(importSchedule.sched) { + let importing = false; + this.importTimer = later.setInterval( () => { + if(!importing) { + importing = true; + + Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message import/toss...'); + + this.performImport( () => { + importing = false; + }); + } + }, importSchedule.sched); + } } } - const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import); - if(importSchedule) { - if(importSchedule.sched) { - let importing = false; - this.importTimer = later.setInterval( () => { - if(!importing) { - importing = true; - - Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message import/toss...'); - - this.performImport(err => { - importing = false; - }); - } - }, importSchedule.sched); - } - } - } - - FTNMessageScanTossModule.super_.prototype.startup.call(this, cb); + FTNMessageScanTossModule.super_.prototype.startup.call(this, cb); + }); }; FTNMessageScanTossModule.prototype.shutdown = function(cb) { @@ -1029,8 +1042,21 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) { if(this.exportTimer) { this.exportTimer.clear(); } - - FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); + + // + // Clean up temp dir/files we created + // + temp.cleanup((err, stats) => { + const fullStats = Object.assign(stats, { exportTemp : this.exportTempDir, importTemp : this.importTempDir } ); + + if(err) { + Log.warn(fullStats, 'Failed cleaning up temporary directories!'); + } else { + Log.trace(fullStats, 'Temporary directories cleaned up'); + } + + FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); + }); }; FTNMessageScanTossModule.prototype.performImport = function(cb) { From 964c53ea9f8fb6bf5ce438e17c4fa33babc07967 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 14 Mar 2016 22:29:41 -0600 Subject: [PATCH 19/27] * Changed scan check to use new System state_flags0 meta to skip already imported/exported msgs * Use moment.js for Message modTimestamp * Remove user_message_status stuff * Add REPLY kludge support @ export * Use TID vs PID kludge @ export (spec) * Start work on @immediate - nearly complete --- core/database.js | 11 --- core/ftn_util.js | 3 +- core/message.js | 105 ++++++++++++----------- core/scanner_tossers/ftn_bso.js | 143 +++++++++++++++++++++++++------- 4 files changed, 165 insertions(+), 97 deletions(-) diff --git a/core/database.js b/core/database.js index 35e9cc87..3baf682a 100644 --- a/core/database.js +++ b/core/database.js @@ -203,17 +203,6 @@ function createMessageBaseTables() { ' UNIQUE(user_id, area_tag)' + ');' ); - - // :TODO: Not currently used - dbs.message.run( - 'CREATE TABLE IF NOT EXISTS user_message_status (' + - ' user_id INTEGER NOT NULL,' + - ' message_id INTEGER NOT NULL,' + - ' status INTEGER NOT NULL,' + - ' UNIQUE(user_id, message_id, status),' + - ' FOREIGN KEY(user_id) REFERENCES user(id)' + - ');' - ); dbs.message.run( `CREATE TABLE IF NOT EXISTS message_area_last_scan ( diff --git a/core/ftn_util.js b/core/ftn_util.js index fdc5e5a6..e46ee12c 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -68,7 +68,8 @@ function getDateFromFtnDateTime(dateTime) { // "27 Feb 15 00:00:03" // // :TODO: Use moment.js here - return (new Date(Date.parse(dateTime))).toISOString(); + return moment(Date.parse(dateTime)); // Date.parse() allows funky formats +// return (new Date(Date.parse(dateTime))).toISOString(); } function getDateTimeString(m) { diff --git a/core/message.js b/core/message.js index 30b72d7c..a9f74b0f 100644 --- a/core/message.js +++ b/core/message.js @@ -1,14 +1,15 @@ /* jslint node: true */ 'use strict'; -var msgDb = require('./database.js').dbs.message; -var wordWrapText = require('./word_wrap.js').wordWrapText; -var ftnUtil = require('./ftn_util.js'); +let msgDb = require('./database.js').dbs.message; +let wordWrapText = require('./word_wrap.js').wordWrapText; +let ftnUtil = require('./ftn_util.js'); -var uuid = require('node-uuid'); -var async = require('async'); -var _ = require('lodash'); -var assert = require('assert'); +let uuid = require('node-uuid'); +let async = require('async'); +let _ = require('lodash'); +let assert = require('assert'); +let moment = require('moment'); module.exports = Message; @@ -24,10 +25,10 @@ function Message(options) { this.subject = options.subject || ''; this.message = options.message || ''; - if(_.isDate(options.modTimestamp)) { - this.modTimestamp = options.modTimestamp; + if(_.isDate(options.modTimestamp) || moment.isMoment(options.modTimestamp)) { + this.modTimestamp = moment(options.modTimestamp); } else if(_.isString(options.modTimestamp)) { - this.modTimestamp = new Date(options.modTimestamp); + this.modTimestamp = moment(options.modTimestamp); } this.viewCount = options.viewCount || 0; @@ -44,11 +45,8 @@ function Message(options) { this.meta = options.meta; } -// this.meta = options.meta || {}; this.hashTags = options.hashTags || []; - var self = this; - this.isValid = function() { // :TODO: validate as much as possible return true; @@ -59,8 +57,8 @@ function Message(options) { }; this.getMessageTimestampString = function(ts) { - ts = ts || new Date(); - return ts.toISOString(); + ts = ts || moment(); + return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ'); }; } @@ -70,13 +68,7 @@ Message.WellKnownAreaTags = { Bulletin : 'local_bulletin', }; -// :TODO: This doesn't seem like a good way to go -- perhaps only for local/user2user, or just use -// a system similar to the "last read" for general areas -Message.Status = { - New : 0, - Read : 1, -}; - +// :TODO: FTN stuff really doesn't belong here - move it elsewhere and/or just use the names directly when needed Message.MetaCategories = { System : 1, // ENiGMA1/2 stuff FtnProperty : 2, // Various FTN network properties, ftn_cost, ftn_origin, ... @@ -86,6 +78,13 @@ Message.MetaCategories = { Message.SystemMetaNames = { LocalToUserID : 'local_to_user_id', LocalFromUserID : 'local_from_user_id', + StateFlags0 : 'state_flags0', // See Message.StateFlags0 +}; + +Message.StateFlags0 = { + None : 0x00000000, + Imported : 0x00000001, // imported from foreign system + Exported : 0x00000002, // exported to foreign system }; Message.FtnPropertyNames = { @@ -152,7 +151,31 @@ Message.getMessageIdsByMetaValue = function(category, name, value, cb) { ); }; -Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) { +Message.getMetaValuesByMessageId = function(messageId, category, name, cb) { + const sql = + `SELECT meta_value + FROM message_meta + WHERE message_id = ? AND meta_category = ? AND meta_name = ?;`; + + msgDb.all(sql, [ messageId, category, name ], (err, rows) => { + if(err) { + return cb(err); + } + + if(0 === rows.length) { + return cb(new Error('No value for category/name')); + } + + // single values are returned without an array + if(1 === rows.length) { + return cb(null, rows[0].meta_value); + } + + cb(null, rows.map(r => r.meta_value)); // map to array of values only + }); +}; + +Message.getMetaValuesByMessageUuid = function(uuid, category, name, cb) { async.waterfall( [ function getMessageId(callback) { @@ -160,32 +183,14 @@ Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, callback(err, messageId); }); }, - function getMetaValue(messageId, callback) { - const sql = - `SELECT meta_value - FROM message_meta - WHERE message_id = ? AND message_category = ? AND meta_name = ?;`; - - msgDb.all(sql, [ messageId, category, name ], (err, rows) => { - if(err) { - return callback(err); - } - - if(0 === rows.length) { - return callback(new Error('No value for category/name')); - } - - // single values are returned without an array - if(1 === rows.length) { - return callback(null, rows[0].meta_value); - } - - callback(null, rows.map(r => r.meta_value)); + function getMetaValues(messageId, callback) { + Message.getMetaValuesByMessageId(messageId, category, name, (err, values) => { + callback(err, values); }); } ], - (err, value) => { - cb(err, value); + (err, values) => { + cb(err, values); } ); }; @@ -254,7 +259,7 @@ Message.prototype.load = function(options, cb) { self.fromUserName = msgRow.from_user_name; self.subject = msgRow.subject; self.message = msgRow.message; - self.modTimestamp = msgRow.modified_timestamp; + self.modTimestamp = moment(msgRow.modified_timestamp); self.viewCount = msgRow.view_count; callback(err); @@ -269,12 +274,6 @@ Message.prototype.load = function(options, cb) { function loadHashTags(callback) { // :TODO: callback(null); - }, - function loadMessageStatus(callback) { - if(options.user) { - // :TODO: Load from user_message_status - } - callback(null); } ], function complete(err) { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index b112d474..fb5d22fe 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -29,13 +29,12 @@ exports.moduleInfo = { }; /* - :TODO: - * Add bundle timer (arcmail) - * Queue until time elapses / fixed time interval - * Pakcets append until >= max byte size - * [if arch type is not empty): Packets -> bundle until max byte size -> repeat process - * NetMail needs explicit isNetMail() check - * NetMail filename / location / etc. is still unknown - need to post on groups & get real answers + :TODO: + * Support (approx) max bundle size + * Support NetMail + * NetMail needs explicit isNetMail() check + * NetMail filename / location / etc. is still unknown - need to post on groups & get real answers + */ exports.getModule = FTNMessageScanTossModule; @@ -242,9 +241,7 @@ function FTNMessageScanTossModule() { message.meta.FtnProperty.ftn_dest_node = options.destAddress.node; message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net; message.meta.FtnProperty.ftn_dest_network = options.destAddress.net; - // :TODO: attr1 & 2 message.meta.FtnProperty.ftn_cost = 0; - message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); // :TODO: Need an explicit isNetMail() check @@ -308,15 +305,15 @@ function FTNMessageScanTossModule() { } message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); - - if(!message.meta.FtnKludge.PID) { - message.meta.FtnKludge.PID = ftnUtil.getProductIdentifier(); - } - - if(!message.meta.FtnKludge.TID) { - // :TODO: Create TID!! - //message.meta.FtnKludge.TID = - } + + // + // According to FSC-0046: + // + // "When a Conference Mail processor adds a TID to a message, it may not + // add a PID. An existing TID should, however, be replaced. TIDs follow + // the same format used for PIDs, as explained above." + // + message.meta.FtnKludge.TID = ftnUtil.getProductIdentifier(); // // Determine CHRS and actual internal encoding name @@ -331,13 +328,32 @@ function FTNMessageScanTossModule() { options.encoding = encoding; // save for later message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding); - // :TODO: FLAGS kludge? - // :TODO: Add REPLY kludge if appropriate - + // :TODO: FLAGS kludge? }; + this.setReplyKludgeFromReplyToMsgId = function(message, cb) { + // + // Look up MSGID kludge for |message.replyToMsgId|, if any. + // If found, we can create a REPLY kludge with the previously + // discovered MSGID. + // + + if(0 === message.replyToMsgId) { + return cb(null); // nothing to do + } + + Message.getMetaValuesByMessageId(message.replyToMsgId, 'FtnKludge', 'MSGID', (err, msgIdVal) => { + assert(_.isString(msgIdVal)); + + if(!err) { + // got a MSGID - create a REPLY + message.meta.FtnKludge.REPLY = msgIdVal; + } + + cb(null); // this method always passes + }); + }; - // :TODO: change to something like isAreaConfigValid // check paths, Addresses, etc. this.isAreaConfigValid = function(areaConfig) { if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) { @@ -415,7 +431,7 @@ function FTNMessageScanTossModule() { }); }; - this.getNodeConfigKeyForUplink = function(uplink) { + this.getNodeConfigKeyByAddress = function(uplink) { // :TODO: sort by least # of '*' & take top? const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { return Address.fromString(addr).isPatternMatch(uplink); @@ -453,14 +469,20 @@ function FTNMessageScanTossModule() { }); } else { callback(null); - } + } }, function loadMessage(callback) { message.load( { uuid : msgUuid }, err => { - if(!err) { - self.prepareMessage(message, exportOpts); + if(err) { + return callback(err); } - callback(err); + + // General preperation + self.prepareMessage(message, exportOpts); + + self.setReplyKludgeFromReplyToMsgId(message, err => { + callback(err); + }); }); }, function createNewPacket(callback) { @@ -502,7 +524,12 @@ function FTNMessageScanTossModule() { } callback(null); }, - function updateStoredMeta(callback) { + function storeStateFlags0Meta(callback) { + message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => { + callback(err); + }); + }, + function storeMsgIdMeta(callback) { // // We want to store some meta as if we had imported // this message for later reference @@ -577,7 +604,7 @@ function FTNMessageScanTossModule() { this.exportMessagesToUplinks = function(messageUuids, areaConfig, cb) { async.each(areaConfig.uplinks, (uplink, nextUplink) => { - const nodeConfigKey = self.getNodeConfigKeyForUplink(uplink); + const nodeConfigKey = self.getNodeConfigKeyByAddress(uplink); if(!nodeConfigKey) { return nextUplink(); } @@ -657,7 +684,6 @@ function FTNMessageScanTossModule() { const newPath = paths.join(outgoingDir, paths.basename(oldPath)); fs.rename(oldPath, newPath, err => { if(err) { - // :TODO: Log this - but move on to the next file Log.warn( { oldPath : oldPath, newPath : newPath }, 'Failed moving temporary bundle file!'); @@ -756,6 +782,10 @@ function FTNMessageScanTossModule() { }); }, function persistImport(callback) { + // mark as imported + message.meta.System.StateFlags0 = Message.StateFlags0.Imported.toString(); + + // save to disc message.persist(err => { callback(err); }); @@ -783,6 +813,8 @@ function FTNMessageScanTossModule() { if(!_.isString(localNetworkName)) { next(new Error('No configuration for this packet')); } else { + + // :TODO: password needs validated - need to determine if it will use the same node config (which can have wildcards) or something else?! next(null); } @@ -1011,6 +1043,10 @@ FTNMessageScanTossModule.prototype.startup = function(cb) { if(exportSchedule.watchFile) { // :TODO: monitor file for changes/existance with gaze } + + if(_.isBoolean(exportSchedule.immediate)) { + this.exportImmediate = exportSchedule.immediate; + } } const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import); @@ -1043,6 +1079,10 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) { this.exportTimer.clear(); } + if(this.importTimer) { + this.importTimer.clear(); + } + // // Clean up temp dir/files we created // @@ -1087,7 +1127,8 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { // Select all messages that have a message_id > our last scan ID. // Additionally exclude messages that have a ftn_attr_flags FtnProperty meta // as those came via import! - // + // + /* const getNewUuidsSql = `SELECT message_id, message_uuid FROM message m @@ -1096,6 +1137,23 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { FROM message_meta WHERE message_id = m.message_id AND meta_category = 'FtnProperty' AND meta_name = 'ftn_attr_flags') = 0 ORDER BY message_id;`; + */ + + // + // Select all messages with a |message_id| > |lastScanId|. + // Additionally exclude messages with the System state_flags0 which will be present for + // imported or already exported messages + // + // NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here! + // + const getNewUuidsSql = + `SELECT message_id, message_uuid + FROM message m + WHERE area_tag = ? AND message_id > ? AND + (SELECT COUNT(message_id) + FROM message_meta + WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0 + ORDER BY message_id;`; var self = this; @@ -1156,5 +1214,26 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { FTNMessageScanTossModule.prototype.record = function(message) { // - // :TODO: If @immediate, we should do something here! + // This module works off schedules, but we do support @immediate for export + // + if(true !== this.exportImmediate || !this.hasValidConfiguration()) { + return; + } + + if(message.isPrivate()) { + // :TODO: support NetMail + } else if(message.areaTag) { + const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag]; + if(!this.isAreaConfigValid(areaConfig)) { + return; + } + + // :TODO: We must share a check to block export with schedule/timer when this is exporting... + // :TODO: Messages must be marked as "exported" else we will export this particular message again later @ schedule/timer + // ...if getNewUuidsSql in performExport checks for MSGID existence also we can omit already-exported messages + + this.exportMessagesToUplinks( [ message.uuid ], areaConfig, err => { + }); + + } }; From d29829a46c04e11558b02eb3b14947dbd20802f5 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 15 Mar 2016 21:44:24 -0600 Subject: [PATCH 20/27] * Implemented @watch rule for import schedule * Implemented @immediate rule for export schedule --- core/scanner_tossers/ftn_bso.js | 101 +++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 35 deletions(-) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index fb5d22fe..6414ee35 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -21,6 +21,7 @@ let fs = require('fs'); let later = require('later'); let temp = require('temp').track(); // track() cleans up temp dir/files for us let assert = require('assert'); +let gaze = require('gaze'); exports.moduleInfo = { name : 'FTN BSO', @@ -1009,6 +1010,21 @@ function FTNMessageScanTossModule() { }); }); }; + + // Starts an export block - returns true if we can proceed + this.exportingStart = function() { + if(!this.exportRunning) { + this.exportRunning = true; + return true; + } + + return false; + }; + + // ends an export block + this.exportingEnd = function() { + this.exportRunning = false; + }; } require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); @@ -1016,6 +1032,22 @@ require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); FTNMessageScanTossModule.prototype.startup = function(cb) { Log.info(`${exports.moduleInfo.name} Scanner/Tosser starting up`); + let importing = false; + + let self = this; + + function tryImportNow(reasonDesc) { + if(!importing) { + importing = true; + + Log.info( { module : exports.moduleInfo.name }, reasonDesc); + + self.performImport( () => { + importing = false; + }); + } + } + this.createTempDirectories(err => { if(err) { Log.warn( { error : err.toStrong() }, 'Failed creating temporary directories!'); @@ -1025,46 +1057,39 @@ FTNMessageScanTossModule.prototype.startup = function(cb) { if(_.isObject(this.moduleConfig.schedule)) { const exportSchedule = this.parseScheduleString(this.moduleConfig.schedule.export); if(exportSchedule) { - if(exportSchedule.sched) { - let exporting = false; + if(exportSchedule.sched && this.exportingStart()) { this.exportTimer = later.setInterval( () => { - if(!exporting) { - exporting = true; - - Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message scan/export...'); - - this.performExport( () => { - exporting = false; - }); - } + + Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message scan/export...'); + + this.performExport( () => { + this.exportingEnd(); + }); }, exportSchedule.sched); } - if(exportSchedule.watchFile) { - // :TODO: monitor file for changes/existance with gaze - } - if(_.isBoolean(exportSchedule.immediate)) { this.exportImmediate = exportSchedule.immediate; } } const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import); - if(importSchedule) { - if(importSchedule.sched) { - let importing = false; + if(importSchedule) { + if(importSchedule.sched) { this.importTimer = later.setInterval( () => { - if(!importing) { - importing = true; - - Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message import/toss...'); - - this.performImport( () => { - importing = false; - }); - } + tryImportNow('Performing scheduled message import/toss...'); }, importSchedule.sched); } + + if(_.isString(importSchedule.watchFile)) { + gaze(importSchedule.watchFile, (err, watcher) => { + watcher.on('all', (event, watchedPath) => { + if(importSchedule.watchFile === watchedPath) { + tryImportNow(`Performing import/toss due to @watch: ${watchedPath} (${event})`); + } + }); + }); + } } } @@ -1227,13 +1252,19 @@ FTNMessageScanTossModule.prototype.record = function(message) { if(!this.isAreaConfigValid(areaConfig)) { return; } - - // :TODO: We must share a check to block export with schedule/timer when this is exporting... - // :TODO: Messages must be marked as "exported" else we will export this particular message again later @ schedule/timer - // ...if getNewUuidsSql in performExport checks for MSGID existence also we can omit already-exported messages - - this.exportMessagesToUplinks( [ message.uuid ], areaConfig, err => { - }); - + + if(this.exportingStart()) { + this.exportMessagesToUplinks( [ message.uuid ], areaConfig, err => { + const info = { uuid : message.uuid, subject : message.subject }; + + if(err) { + Log.warn(info, 'Failed exporting message'); + } else { + Log.info(info, 'Message exported'); + } + + this.exportingEnd(); + }); + } } }; From a49b510f31e8071051a455e9ba8d94eb8e0cb59a Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 19 Mar 2016 21:07:47 -0600 Subject: [PATCH 21/27] Add .eslint.json --- .eslintrc.json | 26 ++++++++++++++++++++++++++ mods/menu.hjson | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 .eslintrc.json diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 00000000..bc7309be --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,26 @@ +{ + "env": { + "es6": true, + "node": true + }, + "extends": "eslint:recommended", + "rules": { + "indent": [ + "error", + "tab" + ], + "linebreak-style": [ + "error", + "unix" + ], + "quotes": [ + "error", + "single" + ], + "semi": [ + "error", + "always" + ], + "comma-dangle": 0 + } +} \ No newline at end of file diff --git a/mods/menu.hjson b/mods/menu.hjson index 4008a121..78ec3f4a 100644 --- a/mods/menu.hjson +++ b/mods/menu.hjson @@ -1200,7 +1200,7 @@ { value: { 1: 0 } action: @method:prevMessage - } + } { value: { 1: 1 } action: @method:nextMessage From 9fa044119b0c8a712c2dfc96efad9a033c68a161 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 19 Mar 2016 22:17:49 -0600 Subject: [PATCH 22/27] Catch exception @ createServer() e.g. if no PK exists for ssh.js --- core/bbs.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/bbs.js b/core/bbs.js index efc04cac..56b24efc 100644 --- a/core/bbs.js +++ b/core/bbs.js @@ -200,7 +200,13 @@ function startListening(cb) { } const moduleInst = new module.getModule(); - const server = moduleInst.createServer(); + let server; + try { + server = moduleInst.createServer(); + } catch(e) { + logger.log.warn(e, 'Exception caught creating server!'); + return; + } // :TODO: handle maxConnections, e.g. conf.maxConnections From b91c9771fc6819a6661db3667cd886150997e074 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 20 Mar 2016 21:34:39 -0600 Subject: [PATCH 23/27] * Updated Message Network docs * .ANS files updated from attributes --- core/servers/ssh.js | 3 +- docs/msg_networks.md | 74 +++++++------ mods/art/CONNECT1.ANS | Bin 2062 -> 2052 bytes mods/art/DOORMANY.ANS | Bin 245 -> 237 bytes mods/art/WELCOME1.ANS | 36 +++--- mods/art/WELCOME2.ANS | Bin 2681 -> 2661 bytes mods/art/demo_edit_text_view.ans | Bin 655 -> 648 bytes mods/art/demo_edit_text_view1.ans | Bin 781 -> 767 bytes mods/art/demo_fse_local_user.ans | Bin 652 -> 628 bytes mods/art/demo_fse_netmail_body.ans | Bin 145 -> 144 bytes mods/art/demo_fse_netmail_footer_edit.ans | Bin 391 -> 389 bytes .../art/demo_fse_netmail_footer_edit_menu.ans | Bin 311 -> 309 bytes mods/art/demo_fse_netmail_header.ans | Bin 566 -> 561 bytes mods/art/demo_fse_netmail_help.ans | Bin 900 -> 886 bytes mods/art/demo_horizontal_menu_view1.ans | Bin 423 -> 416 bytes mods/art/demo_mask_edit_text_view1.ans | Bin 639 -> 625 bytes mods/art/demo_multi_line_edit_text_view1.ans | Bin 1170 -> 1146 bytes mods/art/demo_selection_vm.ans | Bin 183 -> 177 bytes mods/art/demo_spin_and_toggle.ans | Bin 644 -> 633 bytes mods/art/demo_vertical_menu_view1.ans | Bin 219 -> 208 bytes mods/art/menu_prompt.ans | Bin 259 -> 256 bytes mods/art/msg_area_footer_view.ans | Bin 303 -> 301 bytes mods/art/msg_area_list.ans | Bin 208 -> 204 bytes mods/art/msg_area_post_header.ans | Bin 582 -> 577 bytes mods/art/msg_area_view_header.ans | Bin 624 -> 619 bytes mods/art/test.ans | 104 +++++++++--------- mods/themes/luciano_blocktronics/CHANGE.ANS | Bin 2059 -> 2035 bytes mods/themes/luciano_blocktronics/DONE.ANS | Bin 432 -> 424 bytes mods/themes/luciano_blocktronics/IDLELOG.ANS | Bin 411 -> 403 bytes mods/themes/luciano_blocktronics/LASTCALL.ANS | Bin 2146 -> 2122 bytes mods/themes/luciano_blocktronics/LETTER.ANS | Bin 706 -> 691 bytes mods/themes/luciano_blocktronics/MATRIX.ANS | Bin 4821 -> 4797 bytes mods/themes/luciano_blocktronics/MMENU.ANS | Bin 3368 -> 3346 bytes mods/themes/luciano_blocktronics/MNUPRMT.ANS | Bin 285 -> 283 bytes mods/themes/luciano_blocktronics/MSGBODY.ANS | Bin 177 -> 162 bytes mods/themes/luciano_blocktronics/MSGEFTR.ANS | Bin 252 -> 251 bytes mods/themes/luciano_blocktronics/MSGEHDR.ANS | Bin 1587 -> 1578 bytes mods/themes/luciano_blocktronics/MSGEHLP.ANS | Bin 950 -> 934 bytes mods/themes/luciano_blocktronics/MSGEMFT.ANS | Bin 227 -> 226 bytes mods/themes/luciano_blocktronics/MSGLIST.ANS | Bin 2316 -> 2291 bytes mods/themes/luciano_blocktronics/MSGMNU.ANS | Bin 3383 -> 3361 bytes mods/themes/luciano_blocktronics/MSGQUOT.ANS | Bin 485 -> 470 bytes mods/themes/luciano_blocktronics/MSGVFTR.ANS | Bin 217 -> 216 bytes mods/themes/luciano_blocktronics/MSGVHDR.ANS | Bin 1608 -> 1599 bytes mods/themes/luciano_blocktronics/MSGVHLP.ANS | Bin 1115 -> 1098 bytes mods/themes/luciano_blocktronics/NUA.ANS | Bin 2631 -> 2609 bytes mods/themes/luciano_blocktronics/PAUSE.ANS | Bin 221 -> 220 bytes mods/themes/luciano_blocktronics/STATUS.ANS | Bin 4121 -> 4097 bytes mods/themes/luciano_blocktronics/SYSSTAT.ANS | Bin 2885 -> 2860 bytes mods/themes/luciano_blocktronics/TOONODE.ANS | Bin 2179 -> 2157 bytes mods/themes/luciano_blocktronics/USERLOG.ANS | Bin 3252 -> 3228 bytes mods/themes/luciano_blocktronics/USERLST.ANS | Bin 1863 -> 1838 bytes mods/themes/luciano_blocktronics/WHOSON.ANS | Bin 1187 -> 1162 bytes 53 files changed, 115 insertions(+), 102 deletions(-) diff --git a/core/servers/ssh.js b/core/servers/ssh.js index 58fc4e17..974d4bcc 100644 --- a/core/servers/ssh.js +++ b/core/servers/ssh.js @@ -235,7 +235,8 @@ SSHServerModule.prototype.createServer = function() { privateKey : fs.readFileSync(Config.servers.ssh.privateKeyPem), passphrase : Config.servers.ssh.privateKeyPass, ident : 'enigma-bbs-' + enigVersion + '-srv', - // Note that sending 'banner' breaks at least EtherTerm! + + // Note that sending 'banner' breaks at least EtherTerm! debug : function debugSsh(dbgLine) { if(true === Config.servers.ssh.traceConnections) { Log.trace('SSH: ' + dbgLine); diff --git a/docs/msg_networks.md b/docs/msg_networks.md index 17611f0e..9624a522 100644 --- a/docs/msg_networks.md +++ b/docs/msg_networks.md @@ -7,37 +7,45 @@ FTN networks are configured under the `messageNetworks::ftn` section of `config. ### Networks The `networks` section contains a sub section for network(s) you wish you join your board with. Each entry's key name can be referenced elsewhere in `config.hjson` for FTN oriented configurations. -Members: +**Members**: * `localAddress` (required): FTN address of **your local system** -Example: +**Example**: ```hjson { - networks: { - agoranet: { - localAddress: "46:3/102" + messageNetworks: { + ftn: { + networks: { + agoranet: { + localAddress: "46:3/102" + } + } } } } ``` ### Areas -The `areas` section defines a mapping of local **area tags** to a message network (from `networks` described previously), a FTN area tag, and remote uplink address(s). This section can be thought of similar to the *AREAS.BBS* file used by other BBS packages. +The `areas` section describes a mapping of local **area tags** found in your `messageConferences` to a message network (from `networks` described previously), a FTN specific area tag, and remote uplink address(s). This section can be thought of similar to the *AREAS.BBS* file used by other BBS packages. -Members: +When importing, messages will be placed in the local area that matches key under `areas`. + +**Members**: * `network` (required): Associated network from the `networks` section * `tag` (required): FTN area tag * `uplinks`: An array of FTN address uplink(s) for this network -Example: +**Example**: ```hjson { - ftn: { - areas: { - agoranet_bbs: { - network: agoranet - tag: AGN_BBS - uplinks: "46:1/100" + messageNetworks: { + ftn: { + areas: { + agoranet_bbs: { /* found within messageConferences */ + network: agoranet + tag: AGN_BBS + uplinks: "46:1/100" + } } } } @@ -47,7 +55,7 @@ Example: ### BSO Import / Export The scanner/tosser module `ftn_bso` provides **B**inkley **S**tyle **O**utbound (BSO) import/toss & scan/export of messages EchoMail and NetMail messages. Configuration is supplied in `config.hjson` under `scannerTossers::ftn_bso`. -Members: +**Members**: * `defaultZone` (required): Sets the default BSO outbound zone * `defaultNetwork` (optional): Sets the default network name from `messageNetworks::ftn::networks`. **Required if more than one network is defined**. * `paths` (optional): Override default paths set by the system. This section may contain `outbound`, `inbound`, and `secInbound`. @@ -61,22 +69,24 @@ The `nodes` section defines how to export messages for one or more uplinks. A node entry starts with a FTN style address (up to 5D) **as a key** in `config.hjson`. This key may contain wildcard(s) for net/zone/node/point/domain. -Members: +**Members**: * `packetType` (optional): `2`, `2.2`, or `2+`. Defaults to `2+` for modern mailer compatiability * `packetPassword` (optional): Password for the packet * `encoding` (optional): Encoding to use for message bodies; Defaults to `utf-8` * `archiveType` (optional): Specifies the archive type for ArcMail bundles. Must be a valid archiver name such as `zip` (See archiver configuration) -Example: +**Example**: ```hjson { - ftn_bso: { - nodes: { - "46:*: { - packetType: 2+ - packetPassword: mypass - encoding: cp437 - archiveType: zip + scannerTossers: { + ftn_bso: { + nodes: { + "46:*: { + packetType: 2+ + packetPassword: mypass + encoding: cp437 + archiveType: zip + } } } } @@ -86,19 +96,21 @@ Example: #### Scheduling Schedules can be defined for importing and exporting via `import` and `export` under `schedule`. Each entry is allowed a "free form" text and/or special indicators for immediate export or watch file triggers. - * `@immediate`: Currently only makes sense for exporting: A message will be immediately exported if this trigger is defined in a schedule. - * `@watch:/path/to/file`: This trigger watches the path specified for changes and will trigger an import or export when such events occur. + * `@immediate`: A message will be immediately exported if this trigger is defined in a schedule. Only used for `export`. + * `@watch:/path/to/file`: This trigger watches the path specified for changes and will trigger an import or export when such events occur. Only used for `import`. * Free form text can be things like `at 5:00 pm` or `every 2 hours`. See [Later text parsing documentation](http://bunkat.github.io/later/parsers.html#text) for more information. -Example: +**Example**: ```hjson { - ftn_bso: { - schedule: { - import: every 1 hours or @watch:/path/to/watchfile.ext - export: every 1 hours or @immediate + scannerTossers: { + ftn_bso: { + schedule: { + import: every 1 hours or @watch:/path/to/watchfile.ext + export: every 1 hours or @immediate + } } } } diff --git a/mods/art/CONNECT1.ANS b/mods/art/CONNECT1.ANS index b9fd50bfa320809be3346f3cc940958c252a6a43..d1a870bcd2a2b09a4f4b19fcd530769831631c2c 100644 GIT binary patch delta 118 zcmeAZXc177jyAA1F|al^&*hSiHaBvXjyAM5Hp;zo2P6SzOjJ>y=;JbR;?2$3jOmP% zvl%5fzhP2l+$_LS#K;QNWNb7!pG|G@dseo|983a}H5mCP-(%*UoWR6Bc^#v`=o3(pjyAA1F|al^&*kNkjy5-PmX0>GHa5z=a|a{?X7El_QfK6y=;;C^dT%oE za!vloBD=YT(UTFvme~A^NtuzEmrG%?0gKFLb(RoDM&8ZUtTTYrWMTGvE?zFFV8>8r UR|A8|6WLuSFJNbze2rZX0HeAhJOBUy diff --git a/mods/art/DOORMANY.ANS b/mods/art/DOORMANY.ANS index e3fa4a31c4012abb99d927635ab08f6f0fb8aa6e..315004d6f9049e2c706d0be8e569ac633f00a6e5 100644 GIT binary patch delta 43 vcmey$_?A&XI@-Y6#K79vJeLa$CW=RMa!Ca{hB~_%7))$*n>c}W;&D9y>h}wc delta 36 icmaFM_?1yyI@-Y6#K79vJeQXX2~HG_SOab1LD+14x`vUKi>jV9> z=>!}BlLiL4`uI@-Y6#K79vJeQXX3McXzGxAP!tOOFfx)^yUyD;7Zl1fbHf#gS~_dxOi z^It~Z&39R37=fIhtp9-IcebfOvWI;ZkW}N?0wiy7Gy=(J&TI`{F6n5X1=7)G7S7Vq qhStUgxhaWx>8VAz`K86Fyj)Vjj-k%31_qNqa=K08Vx4Txr3V0pYAp`{ diff --git a/mods/art/demo_edit_text_view.ans b/mods/art/demo_edit_text_view.ans index e6f7a2401b4f1e2d3af8bdccfea559a1ff8b9f40..e2b0f6a44b5c44dfa50cc820357c7143821bf855 100644 GIT binary patch delta 40 wcmeBY?O^4SjyAA1F|al^&z;B}GchB5;)RIGnvC9)OBsVEpI~&E%*)gS0Q6xDy8r+H delta 66 zcmeBR?PukdjyAA1F|al^&*h!S6~o9oF)bZPoQ+`Qovg~}4I~Q~gMj2AMi(w#E~#M0 RP-j;IgUNzSo|7Y(3;-G_5AFZ} diff --git a/mods/art/demo_edit_text_view1.ans b/mods/art/demo_edit_text_view1.ans index 12f111e0dcaab3ae1239e34dcee8234676478b12..85b1d887a0f6cdd66e9bae2bca8383e4389347ac 100644 GIT binary patch delta 80 zcmeBW`_IZH9c^H3Vqk4-o;#5{RGbUM&s6{dkIcN1V(Dl@Yh#04tBF+(6L*JB7GyMM eivK#JV* S^Gi~T6cUS46DM;qH30x9#}lXk delta 137 zcmeyu(!)Ao4kPcxIo3epx(hQem-gg|j9yl{y1LrDT+-17xvIV)#=KmFp>(vNwXs1i xSi34ryRmt$0>}`z{QQ#CB89}F)I?q`sbI%YXIBG*$&5^{lLeUACc81|0RVIB8@d1h diff --git a/mods/art/demo_fse_netmail_body.ans b/mods/art/demo_fse_netmail_body.ans index 1c5455734e597e8ee249e5c066ae1e2d6ce3415e..d38bc77be1817ecb113c5e032d4c2d215f8132be 100644 GIT binary patch delta 22 dcmbQpIDwH*I@-Y6#K79vJXh5>#Bd^i697EJ1)=}| delta 23 ecmbQhIFXTGI@-Y6#K79vJXh5>#E^F)UlRa9h6S$x diff --git a/mods/art/demo_fse_netmail_footer_edit.ans b/mods/art/demo_fse_netmail_footer_edit.ans index f5bad3539c0a7ae96592a23f68c7520bc12b6407..5d9aec62d284fc7f7b0df673ce77903d69c9c8b7 100644 GIT binary patch delta 27 icmZo?Ze^YjJuzklCzn*PW2m#Mfx%=!Mvuv1jD`SniwBti delta 34 pcmZo=ZfBkl&B!}3dIc9RmsGH0sI#kq!DK;3x5)~OtdqkT^#GkS2x$NS diff --git a/mods/art/demo_fse_netmail_footer_edit_menu.ans b/mods/art/demo_fse_netmail_footer_edit_menu.ans index 6da50e74130535af8d92b8d82753944ad491b8b8..bac8ffab94ec7da49c58e446d5b21e4093eb890f 100644 GIT binary patch delta 26 icmdnaw3TT>^u(A5PA;in$53Zi1A~bNJtng;8Ug@x0tf^E delta 32 ocmdnWw4G@}G$ZfC=m;)eE~#M0P-j;IgNX;-CSG8j%+9C>0G&(;fdBvi diff --git a/mods/art/demo_fse_netmail_header.ans b/mods/art/demo_fse_netmail_header.ans index 47ef04e67f99308b0466b6d2f441c9f794818056..298ddbf2c958ab4379c0bedaa58a8f16a45de6c2 100644 GIT binary patch delta 49 zcmV-10M7rm1hE8=oROVLk>^2^DFHbG3M-Rw0YQ;)3<@g>8dE`4Lq#w!lfeN;lh6SK HlL!JXr+yDb delta 34 mcmdnUvW;cJ97f)Wb3B2>Q%6SL$!d(2Kr)^&07$N9YytqhVhRcX diff --git a/mods/art/demo_fse_netmail_help.ans b/mods/art/demo_fse_netmail_help.ans index f24025f8487f44f90c2ebfc63600e68e43c4efe4..701a3e74f4842dbe8192cc21e24e42753adfaa43 100644 GIT binary patch delta 113 zcmZo+|Hd|9;lxF`lLZ*%!8m}?XYyP|`^k41J%P0JWG5!0$)!y4lcxglRwj94F6n54 pTm|W9Lu+G$+!_>~6$(#}ODfng)Y;X*V6q^y>tqFHw#i}4dH|+>9|QmZ delta 139 zcmeyy*1|qvAtUd^g}IEplld6s;iM0v50I_EBt3aEqY999ozWA__XU!^@n5nPsvEvmOA)93Jxk diff --git a/mods/art/demo_horizontal_menu_view1.ans b/mods/art/demo_horizontal_menu_view1.ans index 0e486d39ad6d839c89a9273b76cc9d558abfb3e8..9398469e643723539ec1804380364f674ce870b8 100644 GIT binary patch delta 49 zcmZ3^ynvZYI@-Y6#K79vJa-~@93$7n9M6f{+!?thF{*HKNd-HGI=dPeOg3cnm|Vta F2mn0w43Gc- delta 44 wcmZ3$yquX^I@-Y6#K79vJePMOR~$1h7w^POPe$H}o7}HhN{XeU4Xuq0a;+wIIZQkqI$4p? coRMpCG^6F@sf-THTwGj}?=UJ%{>#_|00h_-V*mgE delta 74 zcmey!@}GrUI@-Y6#K79vJePMOS1L0v7w^OZF-G2rZ4N-C UhtYu@$OFO2=NT1%BJUZS0Hlc#T>t<8 diff --git a/mods/art/demo_multi_line_edit_text_view1.ans b/mods/art/demo_multi_line_edit_text_view1.ans index b38c0372ca41265888d7e556d6263fc883d4ccfa..fce6aeab15fcec406d1c907cc8d820a1c984630f 100644 GIT binary patch delta 104 zcmbQl`HO=~I@-Y6#K79vJa-~@BqP_v^eGen*i5!$RG*y4s5H5SQ5TFKFltWbV$zxn Xq8phM3F3*&K(i*lWmcLjz|sT&rN|&O delta 146 zcmeyxF^Q8~I@-Y6#K79vJePMOS0pnp7w^QBDU7@mf7mecPBvjw2a?f@NYRKn@r0c$<+|I@-Y6#K79vJeQY?cOthOKafI(6Ak1Tc_;cc0RYQV2m1g3 diff --git a/mods/art/menu_prompt.ans b/mods/art/menu_prompt.ans index 47b49dab9cb6eaf8ad6e67ba41169c33a84fe26c..bb7ebb4fe13767d1f88e293be4c7c4afe291e905 100644 GIT binary patch delta 23 fcmZo>YG9gRHqqRUi%UA%AXnk;y}S1&HZ}nOSq=#$ delta 27 jcmZo*YG#^X#>hL-%#MebOFG&hSK;oxyZ3k})-?eDX)6g! diff --git a/mods/art/msg_area_footer_view.ans b/mods/art/msg_area_footer_view.ans index b5552e27aff66de7ba4b3d0b0be746b7ae2c78ed..f83f887bf42ffaa5eee315062fe078af46900c40 100644 GIT binary patch delta 12 TcmZ3_w3cZ?^u(BeiEEnxASeY% delta 16 WcmZ3>w4P}~G$ZfC=l~$Gx(NU*dj(c!rTnI@-Y6#K79vJa;0u0wdQ%J^6`#O#oiG2L=ED delta 31 mcmX@Zc!7~yI@-Y6#K79vJePMOmjW{{7w<%Ec}CueUQGablm|8d diff --git a/mods/art/msg_area_post_header.ans b/mods/art/msg_area_post_header.ans index 3959d681b51beba46b7e84c66513b9434aed9209..d9d4f18d00bb90d1cd02728c8779f1a81b95b950 100644 GIT binary patch delta 49 zcmV-10M7r$1i=K5oROVEk>FU9IRQBW3M-R=0YQ;)3<@g>8dE`4Lq#w!lhFZ3li&da HlNbUnv=|S* delta 34 mcmX@ea*Sod97f)WbDV(0y$D9$$!3g}Kr){(07&jX$&QQ>%)DIMlNU0&Pb^?#(&nA~lCcQ@7kmu9 diff --git a/mods/art/test.ans b/mods/art/test.ans index 2ebac4ef..16ae8178 100644 --- a/mods/art/test.ans +++ b/mods/art/test.ans @@ -1,52 +1,52 @@ -You should never see this! - - -... nor this -[?33h - fONT tEST - ~~~~~~~~~ - - | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | A | B | C | D | E | F - ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- - 0 |NUL|  |  |  |  |  |  |  |BS |HT |LF | | |CR |  |   - 1 |  |  |  |  |  |  |  |  |  |  |EOF|ESC|  |  |  |   - 2 | | ! | " | # | $ | % | & | ' | ( | ) | * | + | , | - | . | /  - 3 | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | : | ; | < | = | > | ?  - ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- - 4 | @ | A | B | C | D | E | F | G | H | I | J | K | L | M | N | O  - 5 | P | Q | R | S | T | U | V | W | X | Y | Z | [ | \ | ] | ^ | _  - 6 | ` | a | b | c | d | e | f | g | h | i | j | k | l | m | n | o  - 7 | p | q | r | s | t | u | v | w | x | y | z | { | | | } | ~ |   - ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- - 8 | | | | | | | | | | | | | | | |  - 9 | | | | | | | | | | | | | | | |  - A | | | | | | | | | | | | | | | |  - B | | | | | | | | | | | | | | | |  - ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- - C | | | | | | | | | | | | | | | |  - D | | | | | | | | | | | | | | | |  - E | | | | | | | | | | | | | | | |  - F | | | | | | | | | | | | | | | |  - - - cOLOR tEST - ~~~~~~~~~~ - -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  - -  +You should never see this! + + +... nor this +[?33h + fONT tEST + ~~~~~~~~~ + + | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | A | B | C | D | E | F + ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- + 0 |NUL|  |  |  |  |  |  |  |BS |HT |LF | | |CR |  |   + 1 |  |  |  |  |  |  |  |  |  |  |EOF|ESC|  |  |  |   + 2 | | ! | " | # | $ | % | & | ' | ( | ) | * | + | , | - | . | /  + 3 | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | : | ; | < | = | > | ?  + ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- + 4 | @ | A | B | C | D | E | F | G | H | I | J | K | L | M | N | O  + 5 | P | Q | R | S | T | U | V | W | X | Y | Z | [ | \ | ] | ^ | _  + 6 | ` | a | b | c | d | e | f | g | h | i | j | k | l | m | n | o  + 7 | p | q | r | s | t | u | v | w | x | y | z | { | | | } | ~ |   + ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- + 8 | | | | | | | | | | | | | | | |  + 9 | | | | | | | | | | | | | | | |  + A | | | | | | | | | | | | | | | |  + B | | | | | | | | | | | | | | | |  + ---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+--- + C | | | | | | | | | | | | | | | |  + D | | | | | | | | | | | | | | | |  + E | | | | | | | | | | | | | | | |  + F | | | | | | | | | | | | | | | |  + + + cOLOR tEST + ~~~~~~~~~~ + +  +  +  +  +  +  +  +  +  +  +  +  +  +  +  +  + +  diff --git a/mods/themes/luciano_blocktronics/CHANGE.ANS b/mods/themes/luciano_blocktronics/CHANGE.ANS index 22cd22d80d7e82023decfa409afbbc1b135932df..885b3cc9bc181f04ca6b953688abc2511344fa5c 100644 GIT binary patch delta 93 zcmV-j0HXhk5c3a^Adw*W0OJ#Uz4H+NfHVhTQN36-`w9I+#ufy3JMAelLQBZlZgj{xE><% delta 134 zcmey&-z_jffsuEjf+CP`3<469J%GgRI!4~fk&J~v@)TnYkaS~;0g{WDs)3{|a{-X- zWnKs*^;keE*RU)Dl0mE>_662tAZg4N4lWI@-Y6#K79vJeP}WBEQl^qtuBrvNgDX!nxAXh8E61B3A*#yMO!6z576M SBLk3_wJ}gc;r?Vj#wGyW5f)AW delta 74 zcmZ3%yn&ftI@-Y6#K79vJeQXX2q*F=G4f8-O9c{>vw_4v329y~>1ZPZXX$7|Yh#1l a`wI7Yxuk*}L!Dg>3?>^gdQPrjGynjR6B29y diff --git a/mods/themes/luciano_blocktronics/IDLELOG.ANS b/mods/themes/luciano_blocktronics/IDLELOG.ANS index 8397ffbedd825a03e8577c96670f20961be842ac..bcde1ff7edf6f6712afcb0ed5c3ff1175b5bcb41 100644 GIT binary patch delta 53 zcmbQuJeiqCI@-Y6#K79vJeP}WB7gM6i6Rs01t(67o%mitfJ-{s$iP`T+R)nAAosq) J{mGJyO#qcG5D5SP delta 81 zcmbQtJe!$cI@-Y6#K79vJeQXX2q*GHGxAPMR|OJ1vD&;`Ksi3Oq)m)ijulQ|f3CzmlQPM*Q&WXUBRZICM+ZEo%?9c^fBY><2V zHZel??@ShFQkZN3#F0#ja$qeAs3st|lV<~!?O{?7#xNOZ%txk$lP5DjpIpEyK6w($ VT_F8(a}*m3l@JL`nJ|pkM38E8!@PO!G1ZOe}V=g1_xrzYbv7+Q2ZjJK3JTQ$pFYRWNHEcGEx#r diff --git a/mods/themes/luciano_blocktronics/MATRIX.ANS b/mods/themes/luciano_blocktronics/MATRIX.ANS index 3a196643ca3e68b409e8d3126ce71fabd893dd2a..4e18372379762657be552e9bb01e2e8df65401ee 100644 GIT binary patch delta 136 zcmV;30C)e@CA}pI8(T0tG%!0eFl~_w$dS~nlYs%z1PUPA+mnw0Ba?ap>yu^!z>_Bg z(v$iG)|1f%u9JWUt&=VXtdrXZ@srI6;*+om=aZNU$CF?S*^@d9&65ZXoRg&upOZxn q>XR@J{*x~d@slAD;FAIpPzeeeQ$bZjMKCavVG>4@a1sTRrxGqro;Mo+ delta 161 zcmdn1dR3KMI@-Y6#K79vAeVO{*9k`6iI>(e@=mT}yvWGA*`G;>5y(koJ^&;YSuO#| z&n(w~4KN diff --git a/mods/themes/luciano_blocktronics/MMENU.ANS b/mods/themes/luciano_blocktronics/MMENU.ANS index f0e97e2d5651e4fee476e5b823108bee072a311f..35950215c7a622303bd4999c22260f96f18a8532 100644 GIT binary patch delta 155 zcmZ1>HA!lN9OGnp#&wg=F+O48Qn-D4@*PHr$>*7VY(Bxv&p7!8%m2+cSfv>!e_`vN z+`{fY`3!sAWN(hR$*VZ(CTnumOHC_H8bO`&OJbKD%VaR>BkMS;19PckTm2G)#c?< mxPM1F+Soi-I@;XC8OSoqy?q-fdiyq}=;Q((1)#nOJWT++7&ILK diff --git a/mods/themes/luciano_blocktronics/MNUPRMT.ANS b/mods/themes/luciano_blocktronics/MNUPRMT.ANS index 0e7411163b4558d2ab9bed1b8e4f58460789561e..d25116c0c21d5305de60b83f761c1cd4aefd526e 100644 GIT binary patch delta 12 TcmbQsG@EHc*u-$>iL;vk9tZ^d delta 16 WcmbQuG?!^Y7$fh*FlQh!vk3qxlLZw3 diff --git a/mods/themes/luciano_blocktronics/MSGBODY.ANS b/mods/themes/luciano_blocktronics/MSGBODY.ANS index 78771ade371cdfb9432bd6d994388e43ca490783..2c78dc889b97ba6845254d479d3c1008d2bb25d4 100644 GIT binary patch delta 40 kcmdnUxQJ0fI@-Y6#K79vJXh5>#E=UGs76E@P1J7!0HwVLng9R* delta 55 mcmZ3)xRFs`I@-Y6#K79vJXh5>#E_Q@H&l&?G~%78&;$Slxd>eV diff --git a/mods/themes/luciano_blocktronics/MSGEFTR.ANS b/mods/themes/luciano_blocktronics/MSGEFTR.ANS index f4b4f1ac144629ab6929b2c80170d06cf207b8ec..6472298a75734a414311ab93720fa2ae7c911e4e 100644 GIT binary patch delta 9 Qcmeyv_?vM;<;1Ec02i$UkN^Mx delta 11 Scmey(_=j;qB_r>|$|e9E=mebr diff --git a/mods/themes/luciano_blocktronics/MSGEHDR.ANS b/mods/themes/luciano_blocktronics/MSGEHDR.ANS index 70c36bbec8fa26db7e78beeab495d37646ce6bd7..2687f5ee2e515b74f61561b40513e2f658926a56 100644 GIT binary patch delta 59 zcmV-B0L1^Z45|!}X%!0F+Z$UjZ5vxRHA5R)F*`FbZQt9GfUJ`(0pODk0^*bL0^XC* R1M8E#1cj4A1#XkE1%j|c7;69k delta 75 zcmZ3*vzcc?rZ_Lx?c37P2D#GF=BCcl(T3K>2D$fd^G>W=!^k^Xi}3-Fs+Ckx5vQ zqCt`5Hj^6xKmiJqUI9%43JR030bP?00&A0c0$P*R0yUEh11Jd!8dE`4Lq#w!lR*PV LlTZT$lZXQ@)0-cs delta 132 zcmZ3+zKva3I@-Y6#K79vJeQYCI@;LW`OaOo8MdCJmsx YRIp>Hv#Wu@WJ6~6$qvjclPj3@0C&S8egFUf diff --git a/mods/themes/luciano_blocktronics/MSGEMFT.ANS b/mods/themes/luciano_blocktronics/MSGEMFT.ANS index aaa1ce3c081d8aeb319f9b4cbef111ca0a3011bf..f009f7c320ed8faa2aa13f9231932407a84bf26e 100644 GIT binary patch delta 9 QcmaFN_=s^r!oEC2ui delta 11 ScmaFF_?U4*0weFlgeCwRECe|K diff --git a/mods/themes/luciano_blocktronics/MSGLIST.ANS b/mods/themes/luciano_blocktronics/MSGLIST.ANS index 9a8e6ce2653b7bf0a6e1920b8f93d95374a5d223..911f1f2030dd91fe3cdf1e638635457a2565362e 100644 GIT binary patch delta 179 zcmeAX`YbphM~qA1_HF5CgIwuoa}#IjXhUmbgWTJ4ZK849S3$+b-4%s}bM4NM}F*D}ciRouQaxsgej17slxPkzrNKbfCdezHEZ q+T<|ig%C3wSY#(lvfSVNlZBUYGdmkQ<79sJpOZN`8YXvfGywp5L^b#T delta 249 zcmew?*dsI{M}n72;r4CmXoFnoXmb;1>1ac1V}sn=w|OTvrQw$Q_mYu!@^eN5Ai0o9 znAH%ZVX_{RI1A7;g~^6YB5;NwNJK#aXaUGvpdH2_J96*e0gB(pDh@PJ9%$|XCUI`C z+aSc``(V+3U{aP@4XDDIc_Ay<^vS)fa+BFu?m>BdY>JaPSzj>nZvM#1&InZagY74f M{K(z_Bnvs30Jg_RGXMYp diff --git a/mods/themes/luciano_blocktronics/MSGMNU.ANS b/mods/themes/luciano_blocktronics/MSGMNU.ANS index 7e0ab8c8a317d34a457bca25878d29189c6c7f62..e27fed731dfca52ef127d946b6eb39994b54c8a0 100644 GIT binary patch delta 111 zcmV-#0FeK;8lf7H{gM8yldJ*VlZpbulT8C-ld%J8lNtrDLo^>%NFPC(*p|!D5?&J@w;*&SB*@48_Qh{U$yD5hHq(4spketuc1OS!mFKqw- diff --git a/mods/themes/luciano_blocktronics/MSGQUOT.ANS b/mods/themes/luciano_blocktronics/MSGQUOT.ANS index d313b2285db363b33c5ceb7c86d0c6c5316a64f5..e7382b774a9d24c5e9aab388e3e4815f04e5cfe8 100644 GIT binary patch delta 62 zcmaFLe2rODI@-Y6#K79vC|A`t#E=UDCdzJ_ctM5*#F_Xtmy=5>*fG@E)xcnKA*0*m L21eG&+Zpu$@8%Fa delta 102 zcmcb{{FGT%I@-Y6#K79vC|A`t#E_Q@1x^&*q{ho79c^rus~YBOEFEoVZDM9^Y!2la hg;M@FgcOYeR2UK%j6Y|dH_Ie7A61y diff --git a/mods/themes/luciano_blocktronics/MSGVFTR.ANS b/mods/themes/luciano_blocktronics/MSGVFTR.ANS index 81b10fd1a3ce0e67b283def5d24dfe32b50dd39c..d5133959d098bcb68c25e5e0ddef75815d012661 100644 GIT binary patch delta 22 dcmcb~c!P057$=uhuw$sRtAWA9M6Zdf^Z`RIp>Hv#Wu@#6-`DD+~Zu6$eTH diff --git a/mods/themes/luciano_blocktronics/MSGVHDR.ANS b/mods/themes/luciano_blocktronics/MSGVHDR.ANS index 831868378d025d7ecb2005b6ba826a33ea9169b0..754ccee6ac58d55f690ccc753efd94aea09cb958 100644 GIT binary patch delta 74 zcmV-Q0JZ*Ns|}`Fd$tT(f|Me delta 76 zcmdnbbAo3=rZ_Lx?c37P2D#GF=BCcl(T3K>2D$fd^G>W=!^k^Xi}3-FI6ZP zECqQO3Lx9tARrrCFl`%KH!?#TTQNH`Fm2!5lfMO(lT`+vll2CNlYIw#llljblZpsm llj;aPlPC#TlYt2qlb8uC2?`ohK~+OVFffzR2}qL`3Nhq+Et>!U delta 173 zcmdlea$JO8I@-Y6#K79vJeQXX2q*INF!D~E@r;poavkFrAX&upl96|FG&3V3kQ2kw z3M3!0I08u})=DW}E`{5-6%>GGVGAThIU^*%20@K6ff;lAHt*zSP930<^_)!rSb#C2 diff --git a/mods/themes/luciano_blocktronics/PAUSE.ANS b/mods/themes/luciano_blocktronics/PAUSE.ANS index 53bae43257f4ad4bd92542190856779293f8c86a..09e0051c791ce49e66bb77c54cea538ebd9256b6 100644 GIT binary patch delta 9 Qcmcc1c!zO9)Wql}02KoS6aWAK delta 11 Scmcb^c$aZP6eI7%s3rg!gajb~ diff --git a/mods/themes/luciano_blocktronics/STATUS.ANS b/mods/themes/luciano_blocktronics/STATUS.ANS index 33bc53e56bd07e651783ff6c7d5626501ce4d3f7..f119dfb9776c43e83ecaa00c9bcee70ecbacd970 100644 GIT binary patch delta 131 zcmV-}0DS+MAb}u|Y>{nHk-kBa0aue)0iToj0jiUu0a|GC=?U delta 155 zcmZovn5i%!hmm(;P5_YD;|L`FhBNX`4q==RBtJ8*0+Mr?)&of&=Bq$bk!1sryvVW( zNOrR>1CoYp*MTGl`vD+%f_)8;Ea8|0B-uC@0Lf*Xvw@@|S0j)-#I=x>mrFrGVe)Ej h*~u;3yMbc%JTHNy2Jbx}$;S5+NdD(*0Fw3mO#p2?F1r8# diff --git a/mods/themes/luciano_blocktronics/SYSSTAT.ANS b/mods/themes/luciano_blocktronics/SYSSTAT.ANS index 2f3b70443e9566824def202ca92650c73f7614b9..97beb53d8a39949e2982fb876b065d599cc6d492 100644 GIT binary patch delta 146 zcmV;D0B!%p7OWPKG?6t;k%L5$$Y7Hx0c(?$0e6!P0KWi2CtJ42b+_u2Zxh12u&CY8(T1K8(T9rLmOK$J2NnC-`w9I+#uhRn+Q;o z>=Mw8$Q1(O#GE>A8t A6951J delta 157 zcmZ1@c2sPF2_x@B6JH?F=mI2;CNT0&R%Ofvk`oxqfh0TAMj*L^sTN3@GWP(HBbnO+mO#7`Ds%$?lC*g2VpDRXieQ^(~0OeK>`m={eJXK9=~ zljYjv0@f9iCD|G$&t}V-%*O7-%OxFcZ03Cb&V7YD3il_6vnx!lU=J1GQUHS63b&=B R&CHyE^yK^Oijx^RngC9VDE$Bc delta 153 zcmaDW&@4DXfsuEjf)bE$QUMZaUO-|)JR|R9e#TrNS;N=~Bw3j6R01Tk zm=^&_9+pNR*~@YbNG7wc0FvBnAdw!nEFk%f&556vOFG)v%=!ME`wDjy?(Az?xTMZ=Fv&{#Re(WhgaufShR$eXz1%=7|9I}%$I2J>B k6Sx#78*m;5s`$+L3`pMOx(g(?ak~S_zudAwQi`Vu0Fs(4E&u=k diff --git a/mods/themes/luciano_blocktronics/USERLST.ANS b/mods/themes/luciano_blocktronics/USERLST.ANS index 48ee5b6927ef2d287707d3b9b9ee83e4925b3a2c..fa4e349991301e4c5df17d54dc724eb3f882ecf6 100644 GIT binary patch delta 234 zcmX@kw~miXI@-Y6#K79vAa^2n=ftVECKoWioE*h;Z?Yxx!^zSt5tB<;EGMsH$)7C8 zYE;jqaQ}{Uw7H4%?b}=mw{OFlP|^E$;0nM@puDN`eV`TQ=FT7ih5JC;a>2T!qYXiJ z-se&Pir;~Wn>s_q?{I+}ma72cLKIBC#mXy&Apv&IEz$mTIwjNK3biB3=N delta 282 zcmZ3-cbtz~I@-Y6#K79vAeVO{S0^Ly#L2f9c_-&Gz66qCO!t7K3G+iBDasN75@)dl zlFL}~K_aY1^}JjP_wPtYo0~Y_z71sGzKvi*#qZxisDZG7N=%*a^Kt=cb7!!S!hN7C za=}`qqYXiB0ZJ=?G~R*9m^wpcfTAF+2Du7A0e6sExEd3f8jztdB`_6GCn5_YTs!$c Qs}|5lnrzKLat&J(02aqs5&!@I diff --git a/mods/themes/luciano_blocktronics/WHOSON.ANS b/mods/themes/luciano_blocktronics/WHOSON.ANS index 3b083edf6aa9c6960019991ef4595a12d10fde0e..53575482178fba0aa4d84c25e46d730044adb013 100644 GIT binary patch delta 193 zcmZ3?*~K|QVWOhyME8h^3v4IeGny>O=rB2m(SPzH#;D1E8GR+V6mH*FP>_x`$d!&Z zH*%JaHncW2$i07Oatc$;!PtS{Gvlz%GF#f`@~{KHLBR delta 252 zcmeC;T+BH^fsuEjf+~=3jQ|pJZGprsBSzlIJd6%N(udI>NX}-A0+L@DeWiH06mH*F zP>_x`$d!&ZH*%JaHncW2$i07ucXAX{jRa8o{%xqF39{rHCN-c58<@rLnc%=I2392E rb01{T9XJ6s(F|rHPzEYw4if^azH=X&-9TF*b_2;f%sN1lgQW=odpJo( From 0a0468bb12dd3c2d6cec177d0e961001d03d85d8 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 20 Mar 2016 22:17:33 -0600 Subject: [PATCH 24/27] Updated config docs with example config.hjson --- docs/config.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/config.md b/docs/config.md index 78601cc7..be2cad75 100644 --- a/docs/config.md +++ b/docs/config.md @@ -21,5 +21,90 @@ general: { } ``` +#### A Sample Configuration +Below is a **sample** `config.hjson` illustrating various (but not all!) elements that can be configured / tweaked. + + +```hjson +{ + general: { + boardName: A Sample BBS + } + + defaults: { + theme: super-fancy-theme + } + + preLoginTheme: luciano_blocktronics + + messageConferences: { + local_general: { + name: Local + desc: Local Discussions + default: true + + areas: { + local_enigma_dev: { + name: ENiGMA 1/2 Development + desc: Discussion related to development and features of ENiGMA 1/2! + default: true + } + } + } + + agoranet: { + name: Agoranet + desc: This network is for blatant exploitation of the greatest BBS scene art group ever.. ACiD. + + areas: { + agoranet_bbs: { + name: BBS Discussion + desc: Discussion related to BBSs + } + } + } + } + + messageNetworks: { + ftn: { + areas: { + agoranet_bbs: { /* hey kids, this matches above! */ + + // oh oh oh, and this one pairs up with a network below + network: agoranet + tag: AGN_BBS + uplinks: "46:1/100" + } + } + + networks: { + agoranet: { + localAddress: "46:3/102" + } + } + } + } + + scannerTossers: { + ftn_bso: { + schedule: { + import: every 1 hours or @watch:/home/enigma/bink/watchfile.txt + export: every 1 hours or @immediate + } + + defaultZone: 46 + defaultNetwork: agoranet + + nodes: { + "46:*": { + archiveType: ZIP + encoding: utf8 + } + } + } + } +} +``` + ### Menus TODO: Documentation on menu.hjson, etc. \ No newline at end of file From 4e21901be7f5751e74afa530f3bc6c00fc4f6725 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 22 Mar 2016 22:24:00 -0600 Subject: [PATCH 25/27] * Fix hard line feeds @ FTN import/export * Retain Origin and tear lines in imported messages --- core/ftn_mail_packet.js | 12 ++++++++++- core/message_area.js | 2 ++ core/multi_line_edit_text_view.js | 35 ++++++++++++++----------------- core/scanner_tossers/ftn_bso.js | 4 +++- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index c8312eeb..25d2d6be 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -169,8 +169,10 @@ exports.PacketHeader = PacketHeader; // * Writeup on differences between type 2, 2.2, and 2+: // http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt // -function Packet() { +function Packet(options) { var self = this; + + this.options = options || {}; this.parsePacketHeader = function(packetBuffer, cb) { assert(Buffer.isBuffer(packetBuffer)); @@ -574,6 +576,10 @@ function Packet() { if(messageBodyData.tearLine) { msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine; + + if(self.options.keepTearAndOrigin) { + msg.message += `\r\n${messageBodyData.tearLine}\r\n`; + } } if(messageBodyData.seenBy.length > 0) { @@ -586,6 +592,10 @@ function Packet() { if(messageBodyData.originLine) { msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine; + + if(self.options.keepTearAndOrigin) { + msg.message += `${messageBodyData.originLine}\r\n`; + } } const nextBuf = packetBuffer.slice(read); diff --git a/core/message_area.js b/core/message_area.js index 4d47b31e..97565416 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -69,6 +69,8 @@ function getSortedAvailMessageConferences(client, options) { // Return an *object* of available areas within |confTag| function getAvailableMessageAreasByConfTag(confTag, options) { options = options || {}; + + // :TODO: confTag === "" then find default if(_.has(Config.messageConferences, [ confTag, 'areas' ])) { const areas = Config.messageConferences[confTag].areas; diff --git a/core/multi_line_edit_text_view.js b/core/multi_line_edit_text_view.js index cfc6341f..63c5e20b 100644 --- a/core/multi_line_edit_text_view.js +++ b/core/multi_line_edit_text_view.js @@ -266,24 +266,21 @@ function MultiLineEditTextView(options) { } return lines; }; - - this.getOutputText = function(startIndex, endIndex, includeEol) { - var lines = self.getTextLines(startIndex, endIndex); - - // - // Convert lines to contiguous string -- all expanded - // tabs put back to single '\t' characters. - // - var text = ''; - var re = new RegExp('\\t{1,' + (self.tabWidth) + '}', 'g'); - for(var i = 0; i < lines.length; ++i) { - text += lines[i].text.replace(re, '\t'); - if(includeEol && lines[i].eol) { - text += '\n'; - } - } - return text; - }; + + this.getOutputText = function(startIndex, endIndex, eolMarker) { + let lines = self.getTextLines(startIndex, endIndex); + let text = ''; + var re = new RegExp('\\t{1,' + (self.tabWidth) + '}', 'g'); + + lines.forEach(line => { + text += line.text.replace(re, '\t'); + if(eolMarker && line.eol) { + text += eolMarker; + } + }); + + return text; + } this.getContiguousText = function(startIndex, endIndex, includeEol) { var lines = self.getTextLines(startIndex, endIndex); @@ -1018,7 +1015,7 @@ MultiLineEditTextView.prototype.addText = function(text) { }; MultiLineEditTextView.prototype.getData = function() { - return this.getOutputText(0, this.textLines.length, true); + return this.getOutputText(0, this.textLines.length, '\r\n'); }; MultiLineEditTextView.prototype.setPropertyValue = function(propName, value) { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 6414ee35..b674a833 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -806,7 +806,9 @@ function FTNMessageScanTossModule() { this.importMessagesFromPacketFile = function(packetPath, password, cb) { let packetHeader; - new ftnMailPacket.Packet().read(packetPath, (entryType, entryData, next) => { + const packetOpts = { keepTearAndOrigin : true }; + + new ftnMailPacket.Packet(packetOpts).read(packetPath, (entryType, entryData, next) => { if('header' === entryType) { packetHeader = entryData; From 485dccfe11b5d4025446882726593a85877a1cbf Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 22 Mar 2016 22:29:08 -0600 Subject: [PATCH 26/27] SSH not enabled by default (Req's PK/pass in config) --- core/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config.js b/core/config.js index 7525f147..2fccab82 100644 --- a/core/config.js +++ b/core/config.js @@ -192,7 +192,7 @@ function getDefaultConfig() { }, ssh : { port : 8889, - enabled : true, + enabled : false, // defualt to false as PK/pass in config.hjson are required // // Private key in PEM format From 98e6afa1afc2307db4a17c0f00608138b487712c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 23 Mar 2016 20:59:38 -0600 Subject: [PATCH 27/27] * Don't blow up @ message network record() if no network configured for areaTag * Remove console.log() of message persist; use proper client.log --- core/scanner_tossers/ftn_bso.js | 2 +- mods/msg_area_post_fse.js | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 6414ee35..2cb0d3c7 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -357,7 +357,7 @@ function FTNMessageScanTossModule() { // check paths, Addresses, etc. this.isAreaConfigValid = function(areaConfig) { - if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) { + if(!areaConfig || !_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) { return false; } diff --git a/mods/msg_area_post_fse.js b/mods/msg_area_post_fse.js index 16292cea..292c0eec 100644 --- a/mods/msg_area_post_fse.js +++ b/mods/msg_area_post_fse.js @@ -25,7 +25,7 @@ function AreaPostFSEModule(options) { // we're posting, so always start with 'edit' mode this.editorMode = 'edit'; - this.menuMethods.editModeMenuSave = function(formData, extraArgs) { + this.menuMethods.editModeMenuSave = function() { var msg; async.series( @@ -49,9 +49,13 @@ function AreaPostFSEModule(options) { if(err) { // :TODO:... sooooo now what? } else { - console.log(msg); // :TODO: remove me -- probably log that one was saved, however. + // note: not logging 'from' here as it's part of client.log.xxxx() + self.client.log.info( + { to : msg.toUserName, subject : msg.subject, uuid : msg.uuid }, + 'Message persisted' + ); } - + self.nextMenu(); } );