From 6da7d557f91c21c742dc1e30ccf90e9e271e3141 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 6 Dec 2016 18:58:56 -0700 Subject: [PATCH] * Improvements to ANSI parser * Introduction of storage tags for file bases / areas * Expiration for file web server items * WIP work on clean ANSI (on hold for a bit while other file base stuff is worked on) --- core/acs.js | 11 ++ core/ansi_escape_parser.js | 137 ++++++++++++++++----- core/conf_area_util.js | 4 +- core/config.js | 26 ++-- core/database.js | 1 + core/download_queue.js | 4 +- core/file_area.js | 115 ++++++++++++++--- core/file_area_web.js | 68 ++++++++-- core/file_entry.js | 53 ++++---- core/sauce.js | 9 +- core/scanner_tossers/ftn_bso.js | 2 + core/string_util.js | 211 +++++++++++++++++++++++--------- core/user.js | 16 +++ mods/file_area_filter_edit.js | 25 +++- mods/file_area_list.js | 51 ++++---- oputil.js | 4 +- 16 files changed, 557 insertions(+), 180 deletions(-) diff --git a/core/acs.js b/core/acs.js index 3d2fd678..484407da 100644 --- a/core/acs.js +++ b/core/acs.js @@ -25,6 +25,9 @@ class ACS { } } + // + // Message Conferences & Areas + // hasMessageConfRead(conf) { return this.check(conf.acs, 'read', ACS.Defaults.MessageConfRead); } @@ -33,10 +36,17 @@ class ACS { return this.check(area.acs, 'read', ACS.Defaults.MessageAreaRead); } + // + // File Base / Areas + // hasFileAreaRead(area) { return this.check(area.acs, 'read', ACS.Defaults.FileAreaRead); } + hasFileAreaDownload(area) { + return this.check(area.acs, 'download', ACS.Defaults.FileAreaDownload); + } + getConditionalValue(condArray, memberName) { assert(_.isArray(condArray)); assert(_.isString(memberName)); @@ -65,6 +75,7 @@ ACS.Defaults = { MessageConfRead : 'GM[users]', FileAreaRead : 'GM[users]', + FileAreaDownload : 'GM[users]', }; module.exports = ACS; \ No newline at end of file diff --git a/core/ansi_escape_parser.js b/core/ansi_escape_parser.js index c3df3568..f551cd5b 100644 --- a/core/ansi_escape_parser.js +++ b/core/ansi_escape_parser.js @@ -48,8 +48,10 @@ function ANSIEscapeParser(options) { self.row = Math.max(self.row, 1); self.row = Math.min(self.row, self.termHeight); - self.emit('move cursor', self.column, self.row); - self.rowUpdated(); +// self.emit('move cursor', self.column, self.row); + + self.positionUpdated(); + //self.rowUpdated(); }; self.saveCursorPosition = function() { @@ -63,7 +65,9 @@ function ANSIEscapeParser(options) { self.row = self.savedPosition.row; self.column = self.savedPosition.column; delete self.savedPosition; - self.rowUpdated(); + + self.positionUpdated(); +// self.rowUpdated(); }; self.clearScreen = function() { @@ -71,11 +75,76 @@ function ANSIEscapeParser(options) { self.emit('clear screen'); }; +/* self.rowUpdated = function() { self.emit('row update', self.row + self.scrollBack); + };*/ + + self.positionUpdated = function() { + self.emit('position update', self.row, self.column); }; function literal(text) { + let charCode; + let pos; + let start = 0; + const len = text.length; + + function emitLiteral() { + self.emit('literal', text.slice(start, pos)); + start = pos; + } + + for(pos = 0; pos < len; ++pos) { + charCode = text.charCodeAt(pos) & 0xff; // ensure 8bit clean + + switch(charCode) { + case CR : + emitLiteral(); + + self.column = 1; + + self.positionUpdated(); + break; + + case LF : + emitLiteral(); + + self.row += 1; + + self.positionUpdated(); + break; + + default : + if(self.column > self.termWidth) { + // + // Emit data up to this point so it can be drawn before the postion update + // + emitLiteral(); + + self.column = 1; + self.row += 1; + + self.positionUpdated(); + + + } else { + self.column += 1; + } + break; + } + } + + self.emit('literal', text.slice(start)); + + if(self.column > self.termWidth) { + self.column = 1; + self.row += 1; + self.positionUpdated(); + } + } + + function literal2(text) { var charCode; var len = text.length; @@ -88,29 +157,31 @@ function ANSIEscapeParser(options) { case LF : self.row++; - self.rowUpdated(); + self.positionUpdated(); + //self.rowUpdated(); break; default : // wrap - if(self.column === self.termWidth) { + if(self.column > self.termWidth) { self.column = 1; self.row++; - self.rowUpdated(); + //self.rowUpdated(); + self.positionUpdated(); } else { - self.column++; + self.column += 1; } break; } - if(self.row === 26) { // :TODO: should be termHeight + 1 ? - self.scrollBack++; - self.row--; - self.rowUpdated(); + if(self.row === self.termHeight) { + self.scrollBack += 1; + self.row -= 1; + + self.positionUpdated(); } } - //self.emit('chunk', text); self.emit('literal', text); } @@ -188,10 +259,10 @@ function ANSIEscapeParser(options) { } } - self.reset = function(buffer) { + self.reset = function(input) { self.parseState = { // ignore anything past EOF marker, if any - buffer : buffer.split(String.fromCharCode(0x1a), 1)[0], + buffer : input.split(String.fromCharCode(0x1a), 1)[0], re : /(?:\x1b\x5b)([\?=;0-9]*?)([ABCDHJKfhlmnpsu])/g, stop : false, }; @@ -201,7 +272,11 @@ function ANSIEscapeParser(options) { self.parseState.stop = true; }; - self.parse = function() { + self.parse = function(input) { + if(input) { + self.reset(input); + } + // :TODO: ensure this conforms to ANSI-BBS / CTerm / bansi.txt for movement/etc. var pos; var match; @@ -308,40 +383,45 @@ function ANSIEscapeParser(options) { */ function escape(opCode, args) { - var arg; - var i; - var len; + let arg; switch(opCode) { // cursor up case 'A' : - arg = args[0] || 1; + //arg = args[0] || 1; + arg = isNaN(args[0]) ? 1 : args[0]; self.moveCursor(0, -arg); break; // cursor down case 'B' : - arg = args[0] || 1; + //arg = args[0] || 1; + arg = isNaN(args[0]) ? 1 : args[0]; self.moveCursor(0, arg); break; // cursor forward/right case 'C' : - arg = args[0] || 1; + //arg = args[0] || 1; + arg = isNaN(args[0]) ? 1 : args[0]; self.moveCursor(arg, 0); break; // cursor back/left case 'D' : - arg = args[0] || 1; + //arg = args[0] || 1; + arg = isNaN(args[0]) ? 1 : args[0]; self.moveCursor(-arg, 0); break; case 'f' : // horiz & vertical case 'H' : // cursor position - self.row = args[0] || 1; - self.column = args[1] || 1; - self.rowUpdated(); + //self.row = args[0] || 1; + //self.column = args[1] || 1; + self.row = isNaN(args[0]) ? 1 : args[0]; + self.column = isNaN(args[1]) ? 1 : args[1]; + //self.rowUpdated(); + self.positionUpdated(); break; // save position @@ -356,7 +436,7 @@ function ANSIEscapeParser(options) { // set graphic rendition case 'm' : - for(i = 0, len = args.length; i < len; ++i) { + for(let i = 0, len = args.length; i < len; ++i) { arg = args[i]; if(ANSIEscapeParser.foregroundColors[arg]) { @@ -410,12 +490,13 @@ function ANSIEscapeParser(options) { } } } + break; // m - break; + // :TODO: s, u, K // erase display/screen case 'J' : - // :TODO: Handle others + // :TODO: Handle other 'J' types! if(2 === args[0]) { self.clearScreen(); } diff --git a/core/conf_area_util.js b/core/conf_area_util.js index 6009bb34..5d122cc5 100644 --- a/core/conf_area_util.js +++ b/core/conf_area_util.js @@ -16,8 +16,8 @@ function sortAreasOrConfs(areasOrConfs, type) { let entryB; areasOrConfs.sort((a, b) => { - entryA = a[type]; - entryB = b[type]; + entryA = type ? a[type] : a; + entryB = type ? b[type] : b; if(_.isNumber(entryA.sort) && _.isNumber(entryB.sort)) { return entryA.sort - entryB.sort; diff --git a/core/config.js b/core/config.js index 929cd5a9..975e1d55 100644 --- a/core/config.js +++ b/core/config.js @@ -361,17 +361,21 @@ function getDefaultConfig() { fileNamePatterns: { // These are NOT case sensitive + // FILE_ID.DIZ - https://en.wikipedia.org/wiki/FILE_ID.DIZ shortDesc : [ '^FILE_ID\.DIZ$', '^DESC\.SDI$', '^DESCRIPT\.ION$', '^FILE\.DES$', '$FILE\.SDI$', '^DISK\.ID$' ], - longDesc : [ '^.*\.NFO$', '^README\.1ST$', '^README\.TXT$' ], + // common README filename - https://en.wikipedia.org/wiki/README + longDesc : [ + '^.*\.NFO$', '^README\.1ST$', '^README\.TXT$', '^READ\.ME$', '^README$', '^README\.md$' + ], }, yearEstPatterns: [ // - // Patterns should produce the year in the first submatch - // The year may be YY or YYYY + // Patterns should produce the year in the first submatch. + // The extracted year may be YY or YYYY // '[0-3]?[0-9][\\-\\/\\.][0-3]?[0-9][\\-\\/\\.]((?:[0-9]{2})?[0-9]{2})', // m/d/yyyy, mm-dd-yyyy, etc. "\\B('[1789][0-9])\\b", // eslint-disable-line quotes @@ -385,17 +389,25 @@ function getDefaultConfig() { expireMinutes : 1440, // 1 day }, + // + // File area storage location tag/value pairs. + // Non-absolute paths are relative to |areaStoragePrefix|. + // + storageTags : { + sys_msg_attach : 'msg_attach', + }, + areas: { - message_attachment : { - name : 'Message attachments', - desc : 'File attachments to messages', + systemm_message_attachment : { + name : 'Message attachments', + desc : 'File attachments to messages', + storageTags : 'sys_msg_attach', // may be string or array of strings } } }, eventScheduler : { - events : { trimMessageAreas : { // may optionally use [or ]@watch:/path/to/file diff --git a/core/database.js b/core/database.js index e5496ce4..24ef4941 100644 --- a/core/database.js +++ b/core/database.js @@ -262,6 +262,7 @@ const DB_INIT_TABLE = { area_tag VARCHAR NOT NULL, file_sha1 VARCHAR NOT NULL, file_name, /* FTS @ file_fts */ + storage_tag VARCHAR NOT NULL, desc, /* FTS @ file_fts */ desc_long, /* FTS @ file_fts */ upload_timestamp DATETIME NOT NULL diff --git a/core/download_queue.js b/core/download_queue.js index 7554e885..e1ecc6f8 100644 --- a/core/download_queue.js +++ b/core/download_queue.js @@ -7,7 +7,9 @@ module.exports = class DownloadQueue { constructor(client) { this.client = client; - this.loadFromProperty(client); + if(!Array.isArray(this.client.user.downloadQueue)) { + this.loadFromProperty(client); + } } toggle(fileEntry) { diff --git a/core/file_area.js b/core/file_area.js index bd5cc336..ac0ae0d1 100644 --- a/core/file_area.js +++ b/core/file_area.js @@ -37,7 +37,8 @@ function getAvailableFileAreas(client, options) { options = options || { includeSystemInternal : false }; // perform ACS check per conf & omit system_internal if desired - return _.omit(Config.fileAreas.areas, (area, areaTag) => { + const areasWithTags = _.map(Config.fileBase.areas, (area, areaTag) => Object.assign(area, { areaTag : areaTag } ) ); + return _.omit(Config.fileBase.areas, (area, areaTag) => { if(!options.includeSystemInternal && WellKnownAreaTags.MessageAreaAttach === areaTag) { return true; } @@ -48,21 +49,21 @@ function getAvailableFileAreas(client, options) { function getSortedAvailableFileAreas(client, options) { const areas = _.map(getAvailableFileAreas(client, options), v => v); - sortAreasOrConfs(areas, 'area'); + sortAreasOrConfs(areas); return areas; } function getDefaultFileAreaTag(client, disableAcsCheck) { - let defaultArea = _.findKey(Config.fileAreas, o => o.default); + let defaultArea = _.findKey(Config.fileBase, o => o.default); if(defaultArea) { - const area = Config.fileAreas.areas[defaultArea]; + const area = Config.fileBase.areas[defaultArea]; if(true === disableAcsCheck || client.acs.hasFileAreaRead(area)) { return defaultArea; } } // just use anything we can - defaultArea = _.findKey(Config.fileAreas.areas, (area, areaTag) => { + defaultArea = _.findKey(Config.fileBase.areas, (area, areaTag) => { return WellKnownAreaTags.MessageAreaAttach !== areaTag && (true === disableAcsCheck || client.acs.hasFileAreaRead(area)); }); @@ -70,10 +71,10 @@ function getDefaultFileAreaTag(client, disableAcsCheck) { } function getFileAreaByTag(areaTag) { - const areaInfo = Config.fileAreas.areas[areaTag]; + const areaInfo = Config.fileBase.areas[areaTag]; if(areaInfo) { - areaInfo.areaTag = areaTag; // convienence! - areaInfo.storageDirectory = getAreaStorageDirectory(areaInfo); + areaInfo.areaTag = areaTag; // convienence! + areaInfo.storage = getAreaStorageLocations(areaInfo); return areaInfo; } } @@ -113,8 +114,38 @@ function changeFileAreaWithOptions(client, areaTag, options, cb) { ); } -function getAreaStorageDirectory(areaInfo) { - return paths.join(Config.fileBase.areaStoragePrefix, areaInfo.storageDir || ''); +function getAreaStorageDirectoryByTag(storageTag) { + const storageLocation = (storageTag && Config.fileBase.storageTags[storageTag]); + + return paths.resolve(Config.fileBase.areaStoragePrefix, storageLocation || ''); + + /* + // absolute paths as-is + if(storageLocation && '/' === storageLocation.charAt(0)) { + return storageLocation; + } + + // relative to |areaStoragePrefix| + return paths.join(Config.fileBase.areaStoragePrefix, storageLocation || ''); + */ +} + +function getAreaStorageLocations(areaInfo) { + + const storageTags = Array.isArray(areaInfo.storageTags) ? + areaInfo.storageTags : + [ areaInfo.storageTags || '' ]; + + const avail = Config.fileBase.storageTags; + + return _.compact(storageTags.map(storageTag => { + if(avail[storageTag]) { + return { + storageTag : storageTag, + dir : getAreaStorageDirectoryByTag(storageTag), + }; + } + })); } function getFileEntryPath(fileEntry) { @@ -342,29 +373,28 @@ function updateFileEntry(fileEntry, filePath, cb) { } -function addOrUpdateFileEntry(areaInfo, fileName, options, cb) { +function addOrUpdateFileEntry(areaInfo, storageLocation, fileName, options, cb) { const fileEntry = new FileEntry({ areaTag : areaInfo.areaTag, meta : options.meta, hashTags : options.hashTags, // Set() or Array fileName : fileName, + storageTag : storageLocation.storageTag, }); - const filePath = paths.join(getAreaStorageDirectory(areaInfo), fileName); + const filePath = paths.join(storageLocation.dir, fileName); async.waterfall( [ function processPhysicalFile(callback) { - const stream = fs.createReadStream(filePath); - let byteSize = 0; const sha1 = crypto.createHash('sha1'); const sha256 = crypto.createHash('sha256'); const md5 = crypto.createHash('md5'); const crc32 = new CRC32(); - // :TODO: crc32 + const stream = fs.createReadStream(filePath); stream.on('data', data => { byteSize += data.length; @@ -413,6 +443,58 @@ function addOrUpdateFileEntry(areaInfo, fileName, options, cb) { } function scanFileAreaForChanges(areaInfo, cb) { + const storageLocations = getAreaStorageLocations(areaInfo); + + async.eachSeries(storageLocations, (storageLoc, nextLocation) => { + async.series( + [ + function scanPhysFiles(callback) { + const physDir = storageLoc.dir; + + fs.readdir(physDir, (err, files) => { + if(err) { + return callback(err); + } + + async.eachSeries(files, (fileName, nextFile) => { + const fullPath = paths.join(physDir, fileName); + + fs.stat(fullPath, (err, stats) => { + if(err) { + // :TODO: Log me! + return nextFile(null); // always try next file + } + + if(!stats.isFile()) { + return nextFile(null); + } + + addOrUpdateFileEntry(areaInfo, storageLoc, fileName, { }, err => { + return nextFile(err); + }); + }); + }, err => { + return callback(err); + }); + }); + }, + function scanDbEntries(callback) { + // :TODO: Look @ db entries for area that were *not* processed above + return callback(null); + } + ], + err => { + return nextLocation(err); + } + ); + }, + err => { + return cb(err); + }); +} + +/* +function scanFileAreaForChanges2(areaInfo, cb) { const areaPhysDir = getAreaStorageDirectory(areaInfo); async.series( @@ -454,4 +536,5 @@ function scanFileAreaForChanges(areaInfo, cb) { return cb(err); } ); -} \ No newline at end of file +} +*/ \ No newline at end of file diff --git a/core/file_area_web.js b/core/file_area_web.js index 3975390b..cc5227a2 100644 --- a/core/file_area_web.js +++ b/core/file_area_web.js @@ -28,7 +28,8 @@ const WEB_SERVER_PACKAGE_NAME = 'codes.l33t.enigma.web.server'; class FileAreaWebAccess { constructor() { - this.hashids = new hashids(Config.general.boardName); + this.hashids = new hashids(Config.general.boardName); + this.expireTimers = {}; // hashId->timer } startup(cb) { @@ -37,8 +38,7 @@ class FileAreaWebAccess { async.series( [ function initFromDb(callback) { - // :TODO: Init from DB & register expiration timers - return callback(null); + return self.load(callback); }, function addWebRoute(callback) { const webServer = getServer(WEB_SERVER_PACKAGE_NAME); @@ -66,7 +66,56 @@ class FileAreaWebAccess { } load(cb) { - return cb(null); // :TODO: Load from db + // + // Load entries, register expiration timers + // + FileDb.each( + `SELECT hash_id, expire_timestamp + FROM file_web_serve;`, + (err, row) => { + if(row) { + this.scheduleExpire(row.hash_id, moment(row.expire_timestamp)); + } + }, + err => { + return cb(err); + } + ); + } + + removeEntry(hashId) { + // + // Delete record from DB, and our timer + // + FileDb.run( + `DELETE FROM file_web_serve + WHERE hash_id = ?;`, + [ hashId ] + ); + + delete this.expireTime[hashId]; + } + + scheduleExpire(hashId, expireTime) { + + // remove any previous entry for this hashId + const previous = this.expireTimers[hashId]; + if(previous) { + clearTimeout(previous); + delete this.expireTimers[hashId]; + } + + const timeoutMs = expireTime.diff(moment()); + + if(timeoutMs <= 0) { + setImmediate( () => { + this.removeEntry(hashId); + }); + } else { + this.expireTimers[hashId] = setTimeout( () => { + this.removeEntry(hashId); + }, timeoutMs); + } } loadServedHashId(hashId, cb) { @@ -111,12 +160,9 @@ class FileAreaWebAccess { // // Create a URL such as // https://l33t.codes:44512/f/qFdxyZr - // - // :TODO: build from config - // // Prefer HTTPS over HTTP. Be explicit about the port - // only if required. + // only if non-standard. // let schema; let port; @@ -163,8 +209,8 @@ class FileAreaWebAccess { return cb(err); } - // :TODO: setup tracking of expiration time so we can clean up the entry - + this.scheduleExpire(hashId, options.expireTime); + return cb(null, url); } ); @@ -173,7 +219,7 @@ class FileAreaWebAccess { fileNotFound(resp) { resp.writeHead(404, { 'Content-Type' : 'text/html' } ); - // :TODO: allow custom 404 + // :TODO: allow custom 404 - mods//file_area_web-404.html return resp.end('Not found'); } diff --git a/core/file_entry.js b/core/file_entry.js index 17d2995f..f4e65b51 100644 --- a/core/file_entry.js +++ b/core/file_entry.js @@ -12,7 +12,7 @@ const _ = require('lodash'); const paths = require('path'); const FILE_TABLE_MEMBERS = [ - 'file_id', 'area_tag', 'file_sha1', 'file_name', + 'file_id', 'area_tag', 'file_sha1', 'file_name', 'storage_tag', 'desc', 'desc_long', 'upload_timestamp' ]; @@ -44,6 +44,7 @@ module.exports = class FileEntry { this.hashTags = options.hashTags || new Set(); this.fileName = options.fileName; + this.storageTag = options.storageTag; } load(fileId, cb) { @@ -99,9 +100,9 @@ module.exports = class FileEntry { }, function storeEntry(callback) { fileDb.run( - `REPLACE INTO file (area_tag, file_sha1, file_name, desc, desc_long, upload_timestamp) - VALUES(?, ?, ?, ?, ?, ?);`, - [ self.areaTag, self.fileSha1, self.fileName, self.desc, self.descLong, getISOTimestampString() ], + `REPLACE INTO file (area_tag, file_sha1, file_name, storage_tag, desc, desc_long, upload_timestamp) + VALUES(?, ?, ?, ?, ?, ?, ?);`, + [ self.areaTag, self.fileSha1, self.fileName, self.storageTag, self.desc, self.descLong, getISOTimestampString() ], function inserted(err) { // use non-arrow func for 'this' scope / lastID if(!err) { self.fileId = this.lastID; @@ -132,15 +133,21 @@ module.exports = class FileEntry { ); } - get filePath() { - const areaInfo = Config.fileAreas.areas[this.areaTag]; - if(areaInfo) { - return paths.join( - Config.fileBase.areaStoragePrefix, - areaInfo.storageDir || '', - this.fileName - ); + static getAreaStorageDirectoryByTag(storageTag) { + const storageLocation = (storageTag && Config.fileBase.storageTags[storageTag]); + + // absolute paths as-is + if(storageLocation && '/' === storageLocation.charAt(0)) { + return storageLocation; } + + // relative to |areaStoragePrefix| + return paths.join(Config.fileBase.areaStoragePrefix, storageLocation || ''); + } + + get filePath() { + const storageDir = FileEntry.getAreaStorageDirectoryByTag(this.storageTag); + return paths.join(storageDir, this.fileName); } static persistMetaValue(fileId, name, value, cb) { @@ -193,7 +200,7 @@ module.exports = class FileEntry { static getWellKnownMetaValues() { return Object.keys(FILE_WELL_KNOWN_META); } - static findFiles(criteria, cb) { + static findFiles(filter, cb) { // :TODO: build search here - return [ fileid1, fileid2, ... ] // free form // areaTag @@ -201,6 +208,8 @@ module.exports = class FileEntry { // order by // sort + filter = filter || {}; + let sql = `SELECT file_id FROM file`; @@ -216,21 +225,23 @@ module.exports = class FileEntry { sqlWhere += clause; } - if(criteria.areaTag) { - appendWhereClause(`area_tag="${criteria.areaTag}"`); + if(filter.areaTag) { + appendWhereClause(`area_tag="${filter.areaTag}"`); } - if(criteria.search) { + if(filter.terms) { appendWhereClause( `file_id IN ( SELECT rowid FROM file_fts - WHERE file_fts MATCH "${criteria.search.replace(/"/g,'""')}" + WHERE file_fts MATCH "${filter.terms.replace(/"/g,'""')}" )` ); } - if(Array.isArray(criteria.hashTags)) { + if(filter.tags) { + const tags = filter.tags.split(' '); // filter stores as sep separated values + appendWhereClause( `file_id IN ( SELECT file_id @@ -238,14 +249,14 @@ module.exports = class FileEntry { WHERE hash_tag_id IN ( SELECT hash_tag_id FROM hash_tag - WHERE hash_tag IN (${criteria.hashTags.join(',')}) + WHERE hash_tag IN (${tags.join(',')}) ) )` ); } - // :TODO: criteria.orderBy - // :TODO: criteria.sort + // :TODO: filter.orderBy + // :TODO: filter.sort sql += sqlWhere + ';'; const matchingFileIds = []; diff --git a/core/sauce.js b/core/sauce.js index 0dad1bc9..295a6069 100644 --- a/core/sauce.js +++ b/core/sauce.js @@ -50,20 +50,17 @@ function readSAUCE(data, cb) { .tap(function onVars(vars) { if(!SAUCE_ID.equals(vars.id)) { - cb(new Error('No SAUCE record present')); - return; + return cb(new Error('No SAUCE record present')); } var ver = iconv.decode(vars.version, 'cp437'); if('00' !== ver) { - cb(new Error('Unsupported SAUCE version: ' + ver)); - return; + return cb(new Error('Unsupported SAUCE version: ' + ver)); } if(-1 === SAUCE_VALID_DATA_TYPES.indexOf(vars.dataType)) { - cb(new Error('Unsupported SAUCE DataType: ' + vars.dataType)); - return; + return cb(new Error('Unsupported SAUCE DataType: ' + vars.dataType)); } var sauce = { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index d9b9e13b..8cd6e1a0 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -1098,6 +1098,8 @@ function FTNMessageScanTossModule() { return nextFile(); // unknown archive type } + + Log.debug( { bundleFile : bundleFile }, 'Processing bundle' ); self.archUtil.extractTo( bundleFile.path, diff --git a/core/string_util.js b/core/string_util.js index 0ba8dec9..edb0e613 100644 --- a/core/string_util.js +++ b/core/string_util.js @@ -2,8 +2,12 @@ 'use strict'; // ENiGMA½ -const miscUtil = require('./misc_util.js'); -const iconv = require('iconv-lite'); +const miscUtil = require('./misc_util.js'); +const ANSIEscapeParser = require('./ansi_escape_parser.js').ANSIEscapeParser; +const ANSI = require('./ansi_term.js'); + +// deps +const iconv = require('iconv-lite'); exports.stylizeString = stylizeString; exports.pad = pad; @@ -16,6 +20,7 @@ exports.renderStringLength = renderStringLength; exports.formatByteSizeAbbr = formatByteSizeAbbr; exports.formatByteSize = formatByteSize; exports.cleanControlCodes = cleanControlCodes; +exports.createCleanAnsi = createCleanAnsi; // :TODO: create Unicode verison of this const VOWELS = [ 'a', 'e', 'i', 'o', 'u' ]; @@ -310,15 +315,23 @@ function formatByteSize(byteSize, withAbbr, decimals) { //const REGEXP_ANSI_CONTROL_CODES = /(\x1b\x5b)([\?=;0-9]*?)([0-9A-ORZcf-npsu=><])/g; const REGEXP_ANSI_CONTROL_CODES = /(?:\x1b\x5b)([\?=;0-9]*?)([A-ORZcf-npsu=><])/g; const ANSI_OPCODES_ALLOWED_CLEAN = [ - 'C', 'm' , - 'A', 'B', 'D' + 'A', 'B', // up, down + 'C', 'D', // right, left + 'm', // color ]; -function cleanControlCodes(input) { +const AnsiSpecialOpCodes = { + positioning : [ 'A', 'B', 'C', 'D' ], // up, down, right, left + style : [ 'm' ] // color +}; + +function cleanControlCodes(input, options) { let m; let pos; let cleaned = ''; + options = options || {}; + // // Loop through |input| adding only allowed ESC // sequences and literals to |cleaned| @@ -332,6 +345,10 @@ function cleanControlCodes(input) { cleaned += input.slice(pos, m.index); } + if(options.all) { + continue; + } + if(ANSI_OPCODES_ALLOWED_CLEAN.indexOf(m[2].charAt(0)) > -1) { cleaned += m[0]; } @@ -347,61 +364,141 @@ function cleanControlCodes(input) { return cleaned; } -function getCleanAnsi(input) { - // - // Process |input| and produce |cleaned|, an array - // of lines with "clean" ANSI. - // - // Clean ANSI: - // * Contains only color/SGR sequences - // * All movement (up/down/left/right) removed but positioning - // left intact via spaces/etc. - // - // Temporary processing will occur in a grid. Each cell - // containing a character (defaulting to space) possibly a SGR - // - - let m; - let pos; - let grid = []; - let gridPos = { row : 0, col : 0 }; - - function updateGrid(data, dataType) { - // - // Start at to grid[row][col] and populate val[0]...val[N] - // creating cells as necessary - // - if(!grid[gridPos.row]) { - grid[gridPos.row] = []; - } - - if('literal' === dataType) { - data.forEach(c => { - grid[gridPos.row][gridPos.col] = (grid[gridPos.row][gridPos.col] || '') + c; // append to existing SGR - gridPos.col++; - }); - } else if('sgr' === dataType) { - grid[gridPos.row][gridPos.col] = (grid[gridPos.row][gridPos.col] || '') + data; - } - } - - function literal(s) { - let charCode; - const len = s.length; - for(let i = 0; i < len; ++i) { - charCode = s.charCodeAt(i) & 0xff; +function createCleanAnsi(input, options, cb) { + options.width = options.width || 80; + options.height = options.height || 25; + + const canvas = new Array(options.height); + for(let i = 0; i < options.height; ++i) { + canvas[i] = new Array(options.width); + for(let j = 0; j < options.width; ++j) { + canvas[i][j] = {}; } } - - do { - pos = REGEXP_ANSI_CONTROL_CODES.lastIndex; - m = REGEXP_ANSI_CONTROL_CODES.exec(input); - - if(null !== m) { - if(m.index > pos) { - updateGrid(input.slice(pos, m.index), 'literal'); + + const parserOpts = { + termHeight : options.height, + termWidth : options.width, + }; + + const parser = new ANSIEscapeParser(parserOpts); + + const canvasPos = { + col : 0, + row : 0, + }; + + let sgr; + + function ensureCell() { + // we've pre-allocated a matrix, but allow for > supplied dimens up front. They will be trimmed @ finalize + if(!canvas[canvasPos.row]) { + canvas[canvasPos.row] = new Array(options.width); + for(let j = 0; j < options.width; ++j) { + canvas[canvasPos.row][j] = {}; } } - } while(0 !== REGEXP_ANSI_CONTROL_CODES.lastIndex); + canvas[canvasPos.row][canvasPos.col] = canvas[canvasPos.row][canvasPos.col] || {}; + //canvas[canvasPos.row][0].width = Math.max(canvas[canvasPos.row][0].width || 0, canvasPos.col); + } + + parser.on('literal', literal => { + // + // CR/LF are handled for 'position update'; we don't need the chars themselves + // + literal = literal.replace(/\r?\n|[\r\u2028\u2029]/g, ''); + + for(let i = 0; i < literal.length; ++i) { + const c = literal.charAt(i); + + ensureCell(); + + canvas[canvasPos.row][canvasPos.col].char = c; + + if(sgr) { + canvas[canvasPos.row][canvasPos.col].sgr = sgr; + sgr = null; + } + + canvasPos.col += 1; + } + }); + + parser.on('control', (match, opCode) => { + if('m' !== opCode) { + return; // don't care' + } + sgr = match; + }); + + parser.on('position update', (row, col) => { + canvasPos.row = row - 1; + canvasPos.col = Math.min(col - 1, options.width); + }); + + parser.on('complete', () => { + for(let row = 0; row < options.height; ++row) { + let col = 0; + + //while(col <= canvas[row][0].width) { + while(col < options.width) { + if(!canvas[row][col].char) { + canvas[row][col].char = 'P'; + if(!canvas[row][col].sgr) { + // :TODO: fix duplicate SGR's in a row here - we just need one per sequence + canvas[row][col].sgr = ANSI.reset(); + } + } + + col += 1; + } + + // :TODO: end *all* with CRLF - general usage will be width : 79 - prob update defaults + + if(col <= options.width) { + canvas[row][col] = canvas[row][col] || {}; + + //canvas[row][col].char = '\r\n'; + canvas[row][col].sgr = ANSI.reset(); + + // :TODO: don't splice, just reset + fill with ' ' till end + for(let fillCol = col; fillCol <= options.width; ++fillCol) { + canvas[row][fillCol].char = 'X'; + } + + //canvas[row] = canvas[row].splice(0, col + 1); + //canvas[row][options.width - 1].char = '\r\n'; + + + } else { + canvas[row] = canvas[row].splice(0, options.width + 1); + } + + } + + let out = ''; + for(let row = 0; row < options.height; ++row) { + out += canvas[row].map( col => { + let c = col.sgr || ''; + c += col.char; + return c; + }).join(''); + + } + + // :TODO: finalize: @ any non-char cell, reset sgr & set to ' ' + // :TODO: finalize: after sgr established, omit anything > supplied dimens + return cb(out); + }); + + parser.parse(input); } + +const fs = require('fs'); +let data = fs.readFileSync('/home/nuskooler/Downloads/art3.ans'); +data = iconv.decode(data, 'cp437'); +createCleanAnsi(data, { width : 79, height : 25 }, (out) => { + out = iconv.encode(out, 'cp437'); + fs.writeFileSync('/home/nuskooler/Downloads/art4.ans', out); +}); diff --git a/core/user.js b/core/user.js index 0288d8dd..e7b14740 100644 --- a/core/user.js +++ b/core/user.js @@ -325,6 +325,22 @@ User.prototype.persistProperty = function(propName, propValue, cb) { ); }; +User.prototype.removeProperty = function(propName, cb) { + // update live + delete this.properties[propName]; + + userDb.run( + `DELETE FROM user_property + WHERE user_id = ? AND prop_name = ?;`, + [ this.userId, propName ], + err => { + if(cb) { + return cb(err); + } + } + ) +}; + User.prototype.persistProperties = function(properties, cb) { var self = this; diff --git a/mods/file_area_filter_edit.js b/mods/file_area_filter_edit.js index 7dfc3143..80d87fa3 100644 --- a/mods/file_area_filter_edit.js +++ b/mods/file_area_filter_edit.js @@ -30,6 +30,7 @@ const MciViewIds = { selectedFilterInfo : 10, // { ...filter object ... } activeFilterInfo : 11, // { ...filter object ... } + error : 12, // validation errors } }; @@ -67,7 +68,6 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule { this.updateActiveLabel(); - // :TODO: Need to update %FN somehow return cb(null); }, newFilter : (formData, extraArgs, cb) => { @@ -92,9 +92,8 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule { if(newActive) { filters.setActive(newActive.uuid); } else { - // nothing to set active to - // :TODO: is this what we want? - this.client.user.properties.file_base_filter_active_uuid = 'none'; + // nothing to set active to + this.client.user.removeProperty('file_base_filter_active_uuid'); } } @@ -106,7 +105,23 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule { } return cb(null); }); - } + }, + + viewValidationListener : (err, cb) => { + const errorView = this.viewControllers.editor.getView(MciViewIds.editor.error); + let newFocusId; + + if(errorView) { + if(err) { + errorView.setText(err.message); + err.view.clearText(); // clear out the invalid data + } else { + errorView.clearText(); + } + } + + return cb(newFocusId); + }, }; } diff --git a/mods/file_area_list.js b/mods/file_area_list.js index 193021da..219e4d20 100644 --- a/mods/file_area_list.js +++ b/mods/file_area_list.js @@ -8,12 +8,16 @@ const ansi = require('../core/ansi_term.js'); const theme = require('../core/theme.js'); const FileEntry = require('../core/file_entry.js'); const stringFormat = require('../core/string_format.js'); +const createCleanAnsi = require('../core/string_util.js').createCleanAnsi; const FileArea = require('../core/file_area.js'); const Errors = require('../core/enig_error.js').Errors; const ArchiveUtil = require('../core/archive_util.js'); const Config = require('../core/config.js').config; const DownloadQueue = require('../core/download_queue.js'); const FileAreaWeb = require('../core/file_area_web.js'); +const FileBaseFilters = require('../core/file_base_filter.js'); + +const cleanControlCodes = require('../core/string_util.js').cleanControlCodes; // deps const async = require('async'); @@ -328,15 +332,27 @@ exports.getModule = class FileAreaList extends MenuModule { function populateViews(callback) { if(_.isString(self.currentFileEntry.desc)) { const descView = self.viewControllers.browse.getView(MciViewIds.browse.desc); - if(descView) { - descView.setText(self.currentFileEntry.desc); + if(descView) { + createCleanAnsi( + self.currentFileEntry.desc, + { height : self.client.termHeight, width : descView.dimens.width }, + cleanDesc => { + descView.setText(cleanDesc); + + self.updateQueueIndicator(); + self.populateCustomLabels('browse', 10); + + return callback(null); + } + ); + descView.setText( self.currentFileEntry.desc ); } + } else { + self.updateQueueIndicator(); + self.populateCustomLabels('browse', 10); + + return callback(null); } - - self.updateQueueIndicator(); - self.populateCustomLabels('browse', 10); - - return callback(null); } ], err => { @@ -442,20 +458,6 @@ exports.getModule = class FileAreaList extends MenuModule { ); this.updateCustomLabelsWithFilter( 'browse', 10, [ '{isQueued}' ] ); - /* - const indicatorView = this.viewControllers.browse.getView(MciViewIds.browse.queueToggle); - - if(indicatorView) { - const isQueuedIndicator = this.menuConfig.config.isQueuedIndicator || 'Y'; - const isNotQueuedIndicator = this.menuConfig.config.isNotQueuedIndicator || 'N'; - - indicatorView.setText(stringFormat( - this.dlQueue.isQueued(this.currentFileEntry) ? - isQueuedIndicator : - isNotQueuedIndicator - ) - ); - }*/ } cacheArchiveEntries(cb) { @@ -469,7 +471,7 @@ exports.getModule = class FileAreaList extends MenuModule { return cb(Errors.Invalid('Invalid area tag')); } - const filePath = paths.join(areaInfo.storageDirectory, this.currentFileEntry.fileName); + const filePath = this.currentFileEntry.filePath; const archiveUtil = ArchiveUtil.getInstance(); archiveUtil.listEntries(filePath, this.currentFileEntry.entryInfo.archiveType, (err, entries) => { @@ -574,9 +576,10 @@ exports.getModule = class FileAreaList extends MenuModule { } loadFileIds(cb) { - this.fileListPosition = 0; + this.fileListPosition = 0; + const activeFilter = FileBaseFilters.getActiveFilter(this.client); - FileEntry.findFiles(this.filterCriteria, (err, fileIds) => { + FileEntry.findFiles(activeFilter, (err, fileIds) => { this.fileList = fileIds; return cb(err); }); diff --git a/oputil.js b/oputil.js index c78e7683..b00d6428 100755 --- a/oputil.js +++ b/oputil.js @@ -435,7 +435,7 @@ function handleConfigCommand() { } } -function fileAreaScan(areaTag) { +function fileAreaScan() { async.waterfall( [ function init(callback) { @@ -453,7 +453,7 @@ function fileAreaScan(areaTag) { }, function performScan(fileAreaMod, areaInfo, callback) { fileAreaMod.scanFileAreaForChanges(areaInfo, err => { - + return callback(err); }); } ],