diff --git a/core/config.js b/core/config.js index 78e55c42..211e779d 100644 --- a/core/config.js +++ b/core/config.js @@ -215,10 +215,24 @@ function getDefaultConfig() { zip : { sig : '504b0304', offset : 0, - compressCmd : '7z', - compressArgs : [ 'a', '-tzip', '{archivePath}', '{fileList}' ], - decompressCmd : '7z', - decompressArgs : [ 'e', '-o{extractPath}', '{archivePath}' ] + compress : { + cmd : '7z', + args : [ 'a', '-tzip', '{archivePath}', '{fileList}' ], + }, + decompress : { + cmd : '7z', + args : [ 'e', '-o{extractPath}', '{archivePath}' ] + }, + /* + list : { + cmd : '7z', + args : [ 'l', '{archivePath}' ], + match : '...someregex...' + },*/ + extract : { + cmd : '7z', + args : [ 'x', '-o{extractPath}', '{archivePath}', '{fileList}' ], + }, } }, diff --git a/core/file_area.js b/core/file_area.js index 08c600af..0a684442 100644 --- a/core/file_area.js +++ b/core/file_area.js @@ -5,16 +5,24 @@ const Config = require('./config.js').config; const Errors = require('./enig_error.js').Errors; const sortAreasOrConfs = require('./conf_area_util.js').sortAreasOrConfs; +const FileEntry = require('./file_entry.js'); +const FileDb = require('./database.js').dbs.file; +const ArchiveUtil = require('./archive_util.js'); // deps const _ = require('lodash'); const async = require('async'); +const fs = require('fs'); +const crypto = require('crypto'); +const paths = require('path'); exports.getAvailableFileAreas = getAvailableFileAreas; exports.getSortedAvailableFileAreas = getSortedAvailableFileAreas; exports.getDefaultFileArea = getDefaultFileArea; exports.getFileAreaByTag = getFileAreaByTag; exports.changeFileAreaWithOptions = changeFileAreaWithOptions; +//exports.addOrUpdateFileEntry = addOrUpdateFileEntry; +exports.scanFileAreaForChanges = scanFileAreaForChanges; const WellKnownAreaTags = exports.WellKnownAreaTags = { Invalid : '', @@ -64,7 +72,11 @@ function getDefaultFileArea(client, disableAcsCheck) { } function getFileAreaByTag(areaTag) { - return Config.fileAreas.areas[areaTag]; + const areaInfo = Config.fileAreas.areas[areaTag]; + if(areaInfo) { + areaInfo.areaTag = areaTag; // convienence! + return areaInfo; + } } function changeFileAreaWithOptions(client, areaTag, options, cb) { @@ -101,3 +113,167 @@ function changeFileAreaWithOptions(client, areaTag, options, cb) { } ); } + +function getAreaStorageDirectory(areaInfo) { + return paths.join(Config.fileBase.areaStoragePrefix, areaInfo.storageDir || ''); +} + +function getExistingFileEntriesBySha1(sha1, cb) { + const entries = []; + + FileDb.each( + `SELECT file_id, area_tag + FROM file + WHERE file_sha1=?;`, + [ sha1 ], + (err, fileRow) => { + if(fileRow) { + entries.push({ + fileId : fileRow.file_id, + areaTag : fileRow.area_tag, + }); + } + }, + err => { + return cb(err, entries); + } + ); +} + +function addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb) { + async.series( + [ + function getArchiveFileList(callback) { + // :TODO: get list of files in archive + return callback(null); + } + ], + err => { + return cb(err); + } + ); +} + +function addNewFileEntry(fileEntry, filePath, cb) { + const archiveUtil = ArchiveUtil.getInstance(); + + // :TODO: Use detectTypeWithBuf() once avail - we *just* read some file data + archiveUtil.detectType(filePath, (err, archiveType) => { + if(archiveType) { + return addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb); + } else { + // :TODO:addNewNonArchiveFileEntry + } + }); +} + +function addOrUpdateFileEntry(areaInfo, fileName, options, cb) { + + const fileEntry = new FileEntry({ + areaTag : areaInfo.areaTag, + meta : options.meta, + hashTags : options.hashTags, // Set() or Array + }); + + const filePath = paths.join(getAreaStorageDirectory(areaInfo), fileName); + + async.waterfall( + [ + function processPhysicalFile(callback) { + const stream = fs.createReadStream(filePath); + + let byteSize = 0; + const sha1 = crypto.createHash('sha1'); + const sha256 = crypto.createHash('sha256'); + const md5 = crypto.createHash('md5'); + + + // :TODO: crc32 + + stream.on('data', data => { + byteSize += data.length; + + sha1.update(data); + sha256.update(data); + md5.update(data); + }); + + stream.on('end', () => { + fileEntry.meta.byte_size = byteSize; + + // sha-1 is in basic file entry + fileEntry.fileSha1 = sha1.digest('hex'); + + // others are meta + fileEntry.meta.file_sha256 = sha256.digest('hex'); + fileEntry.meta.file_md5 = md5.digest('hex'); + + return callback(null); + }); + + stream.on('error', err => { + return callback(err); + }); + }, + function fetchExistingEntry(callback) { + getExistingFileEntriesBySha1(fileEntry.fileSha1, (err, existingEntries) => { + return callback(err, existingEntries); + }); + }, + function addOrUpdate(callback, existingEntries) { + if(existingEntries.length > 0) { + + } else { + return addNewFileEntry(fileEntry, filePath, callback); + } + }, + ], + err => { + return cb(err); + } + ); +} + +function scanFileAreaForChanges(areaInfo, cb) { + const areaPhysDir = getAreaStorageDirectory(areaInfo); + + async.series( + [ + function scanPhysFiles(callback) { + fs.readdir(areaPhysDir, (err, files) => { + if(err) { + return callback(err); + } + + async.each(files, (fileName, next) => { + const fullPath = paths.join(areaPhysDir, fileName); + + fs.stat(fullPath, (err, stats) => { + if(err) { + // :TODO: Log me! + return next(null); // always try next file + } + + if(!stats.isFile()) { + return next(null); + } + + addOrUpdateFileEntry(areaInfo, fileName, err => { + + }); + }); + }, err => { + return callback(err); + }); + }); + }, + function scanDbEntries(callback) { + // :TODO: Look @ db entries for area that were *not* processed above + return callback(null); + } + ], + err => { + return cb(err); + } + ); +} \ No newline at end of file diff --git a/core/file_entry.js b/core/file_entry.js index ffafbe85..e4c44d11 100644 --- a/core/file_entry.js +++ b/core/file_entry.js @@ -10,7 +10,7 @@ const _ = require('lodash'); const FILE_TABLE_MEMBERS = [ 'file_id', 'area_tag', 'file_sha1', 'file_name', - 'desc', 'desc_long', 'upload_by_username', 'upload_timestamp' + 'desc', 'desc_long', 'upload_by_username', 'upload_timestamp' // :TODO: remove upload_by_username -- and from database.js, etc. ]; const FILE_WELL_KNOWN_META = { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index b5e65227..d9b9e13b 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -37,6 +37,8 @@ exports.moduleInfo = { * Support NetMail * NetMail needs explicit isNetMail() check * NetMail filename / location / etc. is still unknown - need to post on groups & get real answers + * Validate packet passwords!!!! + => secure vs insecure landing areas */ @@ -49,9 +51,7 @@ function FTNMessageScanTossModule() { let self = this; - this.archUtil = new ArchiveUtil(); - this.archUtil.init(); - + this.archUtil = ArchiveUtil.getInstance(); if(_.has(Config, 'scannerTossers.ftn_bso')) { this.moduleConfig = Config.scannerTossers.ftn_bso; diff --git a/mods/file_area_list.js b/mods/file_area_list.js index a56f0a0c..ae1297d8 100644 --- a/mods/file_area_list.js +++ b/mods/file_area_list.js @@ -67,6 +67,8 @@ exports.getModule = class FileAreaList extends MenuModule { this.filterCriteria = this.filterCriteria || { // :TODO: set area tag - all in current area by default }; + + this.currentFileEntry = new FileEntry(); } enter() { @@ -139,9 +141,7 @@ exports.getModule = class FileAreaList extends MenuModule { function fetchEntryData(callback) { return self.loadFileIds(callback); }, - function loadCurrentFileInfo(callback) { - self.currentFileEntry = new FileEntry(); - + function loadCurrentFileInfo(callback) { self.currentFileEntry.load( self.fileList[ self.fileListPosition ], err => { return callback(err); }); diff --git a/oputil.js b/oputil.js index 11f4fd8f..c78e7683 100755 --- a/oputil.js +++ b/oputil.js @@ -38,6 +38,7 @@ global args: commands: user : user utilities config : config file management + file-area : file area management `, User : @@ -56,11 +57,24 @@ valid args: valid args: --new : generate a new/initial configuration +`, + FileArea : +`usage: oputil.js file-area + +valid args: + --scan AREA_TAG : (re)scan area specified by AREA_TAG for new files ` }; -function printUsage(command) { - console.error(USAGE_HELP[command]); +function printUsageAndSetExitCode(command, exitCode) { + if(_.isUndefined(exitCode)) { + exitCode = ExitCodes.ERROR; + } + process.exitCode = exitCode; + const errMsg = USAGE_HELP[command]; + if(errMsg) { + console.error(errMsg); + } } function initConfig(cb) { @@ -144,8 +158,7 @@ function setAccountStatus(userName, active) { function handleUserCommand() { if(true === argv.help || !_.isString(argv.user) || 0 === argv.user.length) { - process.exitCode = ExitCodes.ERROR; - return printUsage('User'); + return printUsageAndSetExitCode('User', ExitCodes.ERROR); } if(_.isString(argv.password)) { @@ -399,8 +412,7 @@ function askNewConfigQuestions(cb) { function handleConfigCommand() { if(true === argv.help) { - process.exitCode = ExitCodes.ERROR; - return printUsage('Config'); + return printUsageAndSetExitCode('Config', ExitCodes.ERROR); } if(argv.new) { @@ -419,11 +431,51 @@ function handleConfigCommand() { } }); } else { - process.exitCode = ExitCodes.ERROR; - return printUsage('Config'); + return printUsageAndSetExitCode('Config', ExitCodes.ERROR); } } +function fileAreaScan(areaTag) { + async.waterfall( + [ + function init(callback) { + return initConfigAndDatabases(callback); + }, + function getFileArea(callback) { + const fileAreaMod = require('./core/file_area.js'); + + const areaInfo = fileAreaMod.getFileAreaByTag(argv.scan); + if(!areaInfo) { + return callback(new Error('Invalid file area')); + } + + return callback(null, fileAreaMod, areaInfo); + }, + function performScan(fileAreaMod, areaInfo, callback) { + fileAreaMod.scanFileAreaForChanges(areaInfo, err => { + + }); + } + ], + err => { + if(err) { + process.exitCode = ExitCodes.ERROR; + console.error(err.message); + } + } + ); +} + +function handleFileAreaCommand() { + if(true === argv.help) { + return printUsageAndSetExitCode('FileArea', ExitCodes.ERROR); + } + + if(argv.scan) { + return fileAreaScan(argv.scan); + } +} + function main() { process.exitCode = ExitCodes.SUCCESS; @@ -435,8 +487,7 @@ function main() { if(0 === argv._.length || 'help' === argv._[0]) { - printUsage('General'); - process.exit(ExitCodes.SUCCESS); + printUsageAndSetExitCode('General', ExitCodes.SUCCESS); } switch(argv._[0]) { @@ -448,9 +499,12 @@ function main() { handleConfigCommand(); break; + case 'file-area' : + handleFileAreaCommand(); + break; + default: - printUsage(''); - process.exitCode = ExitCodes.BAD_COMMAND; + printUsageAndSetExitCode('', ExitCodes.BAD_COMMAND); } }