* WIP on upload scan/processing

* WIP on user add/edit data to uploads
* Add write access (upload) to area ACS
* Add upload collision handling
* Add upload stats

core/enig_error.js

@@ -27,4 +27,5 @@ exports.Errors = {
 	DoesNotExist		: (reason, reasonCode) 	=> new EnigError('Object does not exist', -33002, reason, reasonCode),
 	AccessDenied		: (reason, reasonCode)	=> new EnigError('Access denied', -32003, reason, reasonCode),
 	Invalid				: (reason, reasonCode)	=> new EnigError('Invalid', -32004, reason, reasonCode),
+	ExternalProcess		: (reason, reasonCode)	=> new EnigError('External process error', -32005, reason, reasonCode),
 };

core/file_area.js

@@ -27,7 +27,7 @@ exports.getDefaultFileAreaTag			= getDefaultFileAreaTag;
 exports.getFileAreaByTag				= getFileAreaByTag;
 exports.getFileEntryPath				= getFileEntryPath;
 exports.changeFileAreaWithOptions		= changeFileAreaWithOptions;
-//exports.addOrUpdateFileEntry			= addOrUpdateFileEntry;
+exports.scanFile						= scanFile;
 exports.scanFileAreaForChanges			= scanFileAreaForChanges;
 
 const WellKnownAreaTags					= exports.WellKnownAreaTags = {
@@ -43,16 +43,18 @@ function getAvailableFileAreas(client, options) {
 	options = options || { };
 
 	//	perform ACS check per conf & omit internal if desired
-	return _.omit(Config.fileBase.areas, (area, areaTag) => {        
-		if(!options.includeSystemInternal && isInternalArea(areaTag)) {
+	const allAreas = _.map(Config.fileBase.areas, (areaInfo, areaTag) => Object.assign(areaInfo, { areaTag : areaTag } ));
+	
+	return _.omit(allAreas, areaInfo => {        
+		if(!options.includeSystemInternal && isInternalArea(areaInfo.areaTag)) {
 			return true;
 		}
 
-		if(options.writeAcs && !client.acs.hasFileAreaWrite(area)) {
+		if(options.writeAcs && !client.acs.hasFileAreaWrite(areaInfo)) {
 			return true;	//	omit
 		}
 
-		return !client.acs.hasFileAreaRead(area);
+		return !client.acs.hasFileAreaRead(areaInfo);
 	});
 }
 
@@ -326,42 +328,16 @@ function populateFileEntryWithArchive(fileEntry, filePath, archiveType, cb) {
 	);
 }
 
-function populateFileEntry(fileEntry, filePath, archiveType, cb) {
+function populateFileEntryNonArchive(fileEntry, filePath, archiveType, cb) {
 	//	:TODO:	implement me!
 	return cb(null);
 }
 
 function addNewFileEntry(fileEntry, filePath, cb) {
-	const archiveUtil = ArchiveUtil.getInstance();
-
 	//	:TODO: Use detectTypeWithBuf() once avail - we *just* read some file data
 
 	async.series(
 		[
-			function populateInfo(callback) {
-				archiveUtil.detectType(filePath, (err, archiveType) => {
-					if(archiveType) {
-						//	save this off
-						fileEntry.meta.archive_type = archiveType;
-
-						populateFileEntryWithArchive(fileEntry, filePath, archiveType, err => {
-							if(err) {
-								populateFileEntry(fileEntry, filePath, err => {
-									//	:TODO: log err
-									return callback(null);	//	ignore err
-								});
-							} else {
-								return callback(null);
-							}
-						});
-					} else {
-						populateFileEntry(fileEntry, filePath, err => {
-							//	:TODO: log err
-							return callback(null);	//	ignore err
-						});
-					}
-				});
-			},
 			function addNewDbRecord(callback) {
 				return fileEntry.persist(callback);
 			}
@@ -376,6 +352,102 @@ function updateFileEntry(fileEntry, filePath, cb) {
 
 }
 
+function scanFile(filePath, options, cb) {
+	
+	if(_.isFunction(options) && !cb) {
+		cb = options;
+		options = {};
+	}
+
+	const fileEntry = new FileEntry({
+		areaTag		: options.areaTag,
+		meta		: options.meta,
+		hashTags	: options.hashTags,	//	Set() or Array
+		fileName	: paths.basename(filePath),
+		storageTag	: options.storageTag,
+	});
+
+	async.waterfall(
+		[
+			function processPhysicalFileGeneric(callback) {			
+				let byteSize	= 0;
+				const sha1		= crypto.createHash('sha1');
+				const sha256	= crypto.createHash('sha256');
+				const md5		= crypto.createHash('md5');
+				const crc32		= new CRC32();
+								
+				const stream = fs.createReadStream(filePath);
+
+				stream.on('data', data => {
+					byteSize += data.length;
+
+					sha1.update(data);
+					sha256.update(data);
+					md5.update(data);
+					crc32.update(data);
+				});
+
+				stream.on('end', () => {
+					fileEntry.meta.byte_size = byteSize;
+
+					//	sha-1 is in basic file entry
+					fileEntry.fileSha1 = sha1.digest('hex');
+
+					//	others are meta
+					fileEntry.meta.file_sha256	= sha256.digest('hex');
+					fileEntry.meta.file_md5 	= md5.digest('hex');
+					fileEntry.meta.file_crc32	= crc32.finalize().toString(16);
+
+					return callback(null);
+				});
+
+				stream.on('error', err => {
+					return callback(err);
+				});
+			},
+			function processPhysicalFileByType(callback) {
+				const archiveUtil = ArchiveUtil.getInstance();
+
+				archiveUtil.detectType(filePath, (err, archiveType) => {
+					if(archiveType) {
+						//	save this off
+						fileEntry.meta.archive_type = archiveType;
+
+						populateFileEntryWithArchive(fileEntry, filePath, archiveType, err => {
+							if(err) {
+								populateFileEntryNonArchive(fileEntry, filePath, err => {
+									//	:TODO: log err
+									return callback(null);	//	ignore err
+								});
+							} else {
+								return callback(null);
+							}
+						});
+					} else {
+						populateFileEntryNonArchive(fileEntry, filePath, err => {
+							//	:TODO: log err
+							return callback(null);	//	ignore err
+						});
+					}
+				});
+			},
+			function fetchExistingEntry(callback) {
+				getExistingFileEntriesBySha1(fileEntry.fileSha1, (err, existingEntries) => {
+					return callback(err, existingEntries);
+				});
+			}
+		], 
+		(err, existingEntries) => {
+			if(err) {
+				return cb(err);
+			}
+
+			return cb(null, fileEntry, existingEntries);
+		}
+	);
+}
+
+/*
 function addOrUpdateFileEntry(areaInfo, storageLocation, fileName, options, cb) {
 	
 	const fileEntry = new FileEntry({
@@ -444,6 +516,7 @@ function addOrUpdateFileEntry(areaInfo, storageLocation, fileName, options, cb)
 		}
 	);
 }
+*/
 
 function scanFileAreaForChanges(areaInfo, cb) {
 	const storageLocations = getAreaStorageLocations(areaInfo);
@@ -472,9 +545,28 @@ function scanFileAreaForChanges(areaInfo, cb) {
 									return nextFile(null);
 								}
 
-								addOrUpdateFileEntry(areaInfo, storageLoc, fileName, { }, err => {
-									return nextFile(err);
-								});
+								scanFile(
+									fullPath,
+									{
+										areaTag		: areaInfo.areaTag,
+										storageTag	: storageLoc.storageTag
+									},
+									(err, fileEntry, existingEntries) => {
+										if(err) {
+											//	:TODO: Log me!!!
+											return nextFile(null);	//	try next anyway
+										}
+
+										if(existingEntries.length > 0) {
+											//	:TODO: Handle duplidates -- what to do here???
+										} else {
+											addNewFileEntry(fileEntry, fullPath, err => {
+												//	pass along error; we failed to insert a record in our DB or something else bad
+												return nextFile(err);
+											});
+										}
+									}
+								);
 							});
 						}, err => {
 							return callback(err);
@@ -495,49 +587,3 @@ function scanFileAreaForChanges(areaInfo, cb) {
 		return cb(err);
 	});
 }
-
-/*
-function scanFileAreaForChanges2(areaInfo, cb) {
-	const areaPhysDir = getAreaStorageDirectory(areaInfo);
-
-	async.series(
-		[
-			function scanPhysFiles(callback) {
-				fs.readdir(areaPhysDir, (err, files) => {
-					if(err) {
-						return callback(err);
-					}
-
-					async.eachSeries(files, (fileName, next) => {
-						const fullPath = paths.join(areaPhysDir, fileName);
-
-						fs.stat(fullPath, (err, stats) => {
-							if(err) {
-								//	:TODO: Log me!
-								return next(null);	//	always try next file
-							}
-
-							if(!stats.isFile()) {
-								return next(null);
-							}
-
-							addOrUpdateFileEntry(areaInfo, fileName, { areaTag : areaInfo.areaTag }, err => {
-								return next(err);
-							});
-						});
-					}, err => {
-						return callback(err);
-					});
-				});
-			},
-			function scanDbEntries(callback) {
-				//	:TODO: Look @ db entries for area that were *not* processed above
-				return callback(null);
-			}
-		],
-		err => {
-			return cb(err);
-		}
-	);
-}
-*/

core/file_entry.js

@@ -213,6 +213,16 @@ module.exports = class FileEntry {
 		);	
 	}
 
+	setHashTags(hashTags) {
+		if(_.isString(hashTags)) {
+			this.hashTags = new Set(hashTags.split(/[\s,]+/));
+		} else if(Array.isArray(hashTags)) {
+			this.hashTags = new Set(hashTags);
+		} else if(hashTags instanceof Set) {
+			this.hashTags = hashTags;
+		}
+	}
+
 	static getWellKnownMetaValues() { return Object.keys(FILE_WELL_KNOWN_META); }
 
 	static findFiles(filter, cb) {

core/menu_util.js

@@ -148,8 +148,7 @@ function getFormConfigByIDAndMap(menuConfig, formId, mciMap, cb) {
 	//
 	if(_.has(formForId, 'mci') || _.has(formForId, 'submit')) {
 		Log.trace('Using generic configuration');
-		cb(null, formForId);
-		return;
+		return cb(null, formForId);
 	}
 
 	cb(new Error('No matching form configuration found for key \'' + mciReqKey + '\''));

core/multi_line_edit_text_view.js

@@ -409,10 +409,10 @@ function MultiLineEditTextView(options) {
 
 	this.insertCharactersInText = function(c, index, col) {
 		self.textLines[index].text = [
-				self.textLines[index].text.slice(0, col),
-				c,
-				self.textLines[index].text.slice(col)
-			].join('');
+			self.textLines[index].text.slice(0, col),
+			c,
+			self.textLines[index].text.slice(col)
+		].join('');
 
 		//self.cursorPos.col++;
 		self.cursorPos.col += c.length;

core/string_util.js

@@ -444,7 +444,7 @@ function createCleanAnsi(input, options, cb) {
 			//while(col <= canvas[row][0].width) {
 			while(col < options.width) {
 				if(!canvas[row][col].char) {
-					canvas[row][col].char = 'P';
+					canvas[row][col].char = ' ';
 					if(!canvas[row][col].sgr) {
 						//	:TODO: fix duplicate SGR's in a row here - we just need one per sequence
 						canvas[row][col].sgr = ANSI.reset();
@@ -459,12 +459,12 @@ function createCleanAnsi(input, options, cb) {
 			if(col <= options.width) {
 				canvas[row][col] = canvas[row][col] || {};
 
-				//canvas[row][col].char = '\r\n';
+				canvas[row][col].char = '\r\n';
 				canvas[row][col].sgr = ANSI.reset();
 
 				//	:TODO: don't splice, just reset + fill with ' ' till end
 				for(let fillCol = col; fillCol <= options.width; ++fillCol) {
-					canvas[row][fillCol].char = 'X';
+					canvas[row][fillCol].char = ' ';
 				}
 				
 				//canvas[row] = canvas[row].splice(0, col + 1);

core/transfer_file.js

@@ -100,18 +100,15 @@ exports.getModule = class TransferFileModule extends MenuModule {
 		this.sentFileIds = [];
 	}
 
-	get isSending() {
-		return 'send' === this.direction;
+	isSending() {
+		return ('send' === this.direction);
 	}
 
-	restorePipeAfterExternalProc(pipe) {
+	restorePipeAfterExternalProc() {
 		if(!this.pipeRestored) {
 			this.pipeRestored = true;
 
 			this.client.restoreDataHandler();
-			
-			//this.client.term.output.unpipe(pipe);
-			//this.client.term.output.resume();
 		}
 	}
 
@@ -154,16 +151,62 @@ exports.getModule = class TransferFileModule extends MenuModule {
 		}		
 	}
 
+	moveFileWithCollisionHandling(src, dst, cb) {
+		//
+		//	Move |src| -> |dst| renaming to file(1).ext, file(2).ext, etc. 
+		//	in the case of collisions.
+		//
+		const dstPath		= paths.dirname(dst);
+		const dstFileExt	= paths.extname(dst);
+		const dstFileSuffix	= paths.basename(dst, dstFileExt);
+
+		let renameIndex		= 0;
+		let movedOk			= false;
+		let tryDstPath;
+
+		async.until(
+			() => movedOk,	//	until moved OK
+			(cb) => {
+				if(0 === renameIndex) {
+					//	try originally supplied path first
+					tryDstPath = dst;
+				} else {
+					tryDstPath = paths.join(dstPath, `${dstFileSuffix}(${renameIndex})${dstFileExt}`);
+				}
+
+				fse.move(src, tryDstPath, err => {
+					if(err) {
+						if('EEXIST' === err.code) {
+							renameIndex += 1;
+							return cb(null);	//	keep trying
+						}
+
+						return cb(err);
+					}
+
+					movedOk = true;
+					return cb(null, tryDstPath);
+				});
+			},
+			(err, finalPath) => {
+				return cb(err, finalPath);
+			}
+		);
+	}
+
 	recvFiles(cb) {
 		this.executeExternalProtocolHandlerForRecv( (err, tempWorkingDir) => {
 			if(err) {
 				return cb(err);
 			}
 
-			this.receivedFiles = [];
+			this.recvFilePaths = [];
 
 			if(this.recvFileName) {
 				//	file name specified - we expect a single file in |tempWorkingDir|
+				
+				//	:TODO: support non-blind: Move file to dest path, add to recvFilePaths, etc.
+
 				return cb(null);
 			} else {
 				//
@@ -176,19 +219,19 @@ exports.getModule = class TransferFileModule extends MenuModule {
 					}
 
 					async.each(files, (file, nextFile) => {
-						fse.move(
+						this.moveFileWithCollisionHandling(
 							paths.join(tempWorkingDir, file),
 							paths.join(this.recvDirectory, file),
-							err => {
+							(err, destPath) => {
 								if(err) {
-									//	:TODO: IMPORTANT: Handle collisions - rename to FILE(1).EXT, etc.
 									this.client.log.warn(
 										{ tempWorkingDir : tempWorkingDir, recvDirectory : this.recvDirectory, file : file, error : err.message }, 
 										'Failed to move upload file to destination directory'
 									);
 								} else {
-									this.receivedFiles.push(file);
+									this.recvFilePaths.push(destPath);
 								}
+
 								return nextFile(null);	//	don't pass along err; try next
 							}
 						);
@@ -324,16 +367,16 @@ exports.getModule = class TransferFileModule extends MenuModule {
 		});
 
 		externalProc.once('close', () => {
-			return this.restorePipeAfterExternalProc(externalProc);
+			return this.restorePipeAfterExternalProc();
 		});
 
 		externalProc.once('exit', (exitCode) => {
 			this.client.log.debug( { cmd : cmd, args : args, exitCode : exitCode }, 'Process exited' );
 			
-			this.restorePipeAfterExternalProc(externalProc);
+			this.restorePipeAfterExternalProc();
 			externalProc.removeAllListeners();
 
-			return cb(null);
+			return cb(exitCode ? Errors.ExternalProcess(`Process exited with exit code ${exitCode}`, 'EBADEXIT') : null);
 		});	
 	}
 
@@ -366,7 +409,11 @@ exports.getModule = class TransferFileModule extends MenuModule {
 	}
 
 	getMenuResult() {
-		return { sentFileIds : this.sentFileIds };
+		if(this.isSending()) {
+			return { sentFileIds : this.sentFileIds };
+		} else {
+			return { recvFilePaths : this.recvFilePaths };
+		}		
 	}
 
 	updateSendStats(cb) {
@@ -383,9 +430,8 @@ exports.getModule = class TransferFileModule extends MenuModule {
 				fileIds.push(queueItem.fileId);
 			}
 
-			downloadCount += 1;
-
 			if(_.isNumber(queueItem.byteSize)) {
+				downloadCount += 1;
 				downloadBytes += queueItem.byteSize;
 				return next(null);
 			}
@@ -395,6 +441,7 @@ exports.getModule = class TransferFileModule extends MenuModule {
 				if(err) {
 					this.client.log.warn( { error : err.message, path : queueItem.path }, 'File stat failed' );
 				} else {
+					downloadCount += 1;
 					downloadBytes += stats.size;
 				}
 
@@ -416,8 +463,30 @@ exports.getModule = class TransferFileModule extends MenuModule {
 	}
 	
 	updateRecvStats(cb) {
-		//	:TODO: update user & system upload stats
-		return cb(null);
+		let uploadBytes	= 0;
+		let uploadCount	= 0;
+
+		async.each(this.recvFilePaths, (filePath, next) => {
+			//	we just have a path - figure it out
+			fs.stat(filePath, (err, stats) => {
+				if(err) {
+					this.client.log.warn( { error : err.message, path : filePath }, 'File stat failed' );
+				} else {
+					uploadCount	+= 1;
+					uploadBytes += stats.size;
+				}
+
+				return next(null);
+			});
+		}, () => {
+			StatLog.incrementUserStat(this.client.user, 'ul_total_count', uploadCount);
+			StatLog.incrementUserStat(this.client.user, 'ul_total_bytes', uploadBytes);
+			StatLog.incrementSystemStat('ul_total_count', uploadCount);
+			StatLog.incrementSystemStat('ul_total_bytes', uploadBytes);
+
+
+			return cb(null);
+		});
 	}
 
 	initSequence() {
@@ -428,7 +497,7 @@ exports.getModule = class TransferFileModule extends MenuModule {
 		async.series(
 			[
 				function validateConfig(callback) {
-					if(self.isSending) {
+					if(self.isSending()) {
 						if(!Array.isArray(self.sendQueue)) {
 							self.sendQueue = [ self.sendQueue ];  
 						}
@@ -437,7 +506,7 @@ exports.getModule = class TransferFileModule extends MenuModule {
 					return callback(null);
 				},
 				function transferFiles(callback) {
-					if(self.isSending) {
+					if(self.isSending()) {
 						self.sendFiles( err => {
 							if(err) {
 								return callback(err);
@@ -475,7 +544,7 @@ exports.getModule = class TransferFileModule extends MenuModule {
 					});
 				},
 				function updateUserAndSystemStats(callback) {
-					if(self.isSending) {
+					if(self.isSending()) {
 						return self.updateSendStats(callback);
 					} else {
 						return self.updateRecvStats(callback);
@@ -488,9 +557,10 @@ exports.getModule = class TransferFileModule extends MenuModule {
 				}
 
 				//	Wait for a key press - attempt to avoid issues with some terminals after xfer
-				self.client.term.write('|00\nTransfer(s) complete. Press a key\n');
+				//	:TODO: display ANSI if it exists else prompt -- look @ Obv/2 for filename
+				self.client.term.pipeWrite('|00|07\nTransfer(s) complete. Press a key\n');
 				self.client.waitForKeyPress( () => {
-					self.prevMenu();
+					return self.prevMenu();
 				});
 			}
 		);