moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

+ Add compression for larger exports 

* Temp disable of idle monitor while building large lists
* Fix hash tags
* Handle no results & other errors
This commit is contained in:
Bryan Ashby 2018-03-04 09:17:49 -07:00
parent 44a4a4aeb4
commit 74b9d587c9
1 changed files with 96 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
core/file_base_list_export.js
View File

@ -26,6 +26,7 @@ const paths = require('path');
const iconv = require('iconv-lite'); const iconv = require('iconv-lite');
const moment = require('moment'); const moment = require('moment');
const uuidv4 = require('uuid/v4'); const uuidv4 = require('uuid/v4');
const yazl = require('yazl');
/* /*
:TODO: document: :TODO: document:
@ -71,6 +72,7 @@ exports.getModule = class FileBaseListExport extends MenuModule {
this.config.tsFormat = this.config.tsFormat || this.client.currentTheme.helpers.getDateTimeFormat('short'); this.config.tsFormat = this.config.tsFormat || this.client.currentTheme.helpers.getDateTimeFormat('short');
this.config.descWidth = this.config.descWidth || 45; // ie FILE_ID.DIZ this.config.descWidth = this.config.descWidth || 45; // ie FILE_ID.DIZ
this.config.progBarChar = (this.config.progBarChar || '▒').charAt(0); this.config.progBarChar = (this.config.progBarChar || '▒').charAt(0);
this.config.compressThreshold = this.config.compressThreshold || (1440000); // >= 1.44M by default :)
} }
mciReady(mciData, cb) { mciReady(mciData, cb) {
@ -85,6 +87,13 @@ exports.getModule = class FileBaseListExport extends MenuModule {
(callback) => this.prepareList(callback), (callback) => this.prepareList(callback),
], ],
err => { err => {
if(err) {
if('NORESULTS' === err.reasonCode) {
return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'fileBaseExportListNoResults');
}
return this.prevMenu();
}
return cb(err); return cb(err);
} }
); );
@ -157,7 +166,10 @@ exports.getModule = class FileBaseListExport extends MenuModule {
updateStatus('Gathering files for supplied criteria'); updateStatus('Gathering files for supplied criteria');
FileEntry.findFiles(filterCriteria, (err, fileIds) => { FileEntry.findFiles(filterCriteria, (err, fileIds) => {
// :TODO: handle empty file IDs -- bail early. if(0 === fileIds.length) {
return callback(Errors.General('No results for criteria', 'NORESULTS'));
}
return callback(err, headerTemplate, entryTemplate, descIndent, fileIds); return callback(err, headerTemplate, entryTemplate, descIndent, fileIds);
}); });
}, },
@ -166,12 +178,15 @@ exports.getModule = class FileBaseListExport extends MenuModule {
totalFileCount : fileIds.length, totalFileCount : fileIds.length,
}; };
const fileInfo = new FileEntry();
let current = 0; let current = 0;
let listBody = ''; let listBody = '';
const totals = { fileCount : fileIds.length, bytes : 0 }; const totals = { fileCount : fileIds.length, bytes : 0 };
// this may take quite a while; temp disable of idle monitor
self.client.stopIdleMonitor();
async.eachSeries(fileIds, (fileId, nextFileId) => { async.eachSeries(fileIds, (fileId, nextFileId) => {
const fileInfo = new FileEntry();
current += 1; current += 1;
fileInfo.load(fileId, err => { fileInfo.load(fileId, err => {
@ -237,6 +252,9 @@ exports.getModule = class FileBaseListExport extends MenuModule {
} }
}); });
}, err => { }, err => {
// re-enable idle monitor
self.client.startIdleMonitor();
return callback(err, listBody, headerTemplate, totals); return callback(err, listBody, headerTemplate, totals);
}); });
}, },
@ -283,52 +301,56 @@ exports.getModule = class FileBaseListExport extends MenuModule {
const outputFileName = paths.join( const outputFileName = paths.join(
sysTempDownloadDir, sysTempDownloadDir,
`file_list_${uuidv4()}.txt` `file_list_${uuidv4().substr(-8)}_${moment().format('YYYY-MM-DD')}.txt`
); );
fs.writeFile(outputFileName, listBody, 'utf8', err => { fs.writeFile(outputFileName, listBody, 'utf8', err => {
return callback(err, outputFileName, sysTempDownloadArea); if(err) {
return callback(err);
}
self.getSizeAndCompressIfMeetsSizeThreshold(outputFileName, (err, finalOutputFileName, fileSize) => {
return callback(err, finalOutputFileName, fileSize, sysTempDownloadArea);
});
}); });
}); });
}, },
function persistFileEntry(outputFileName, sysTempDownloadArea, callback) { function persistFileEntry(outputFileName, fileSize, sysTempDownloadArea, callback) {
fse.stat(outputFileName, (err, stats) => { const newEntry = new FileEntry({
const newEntry = new FileEntry({ areaTag : sysTempDownloadArea.areaTag,
areaTag : sysTempDownloadArea.areaTag, fileName : paths.basename(outputFileName),
fileName : paths.basename(outputFileName), storageTag : sysTempDownloadArea.storageTags[0],
storageTag : sysTempDownloadArea.storageTags[0], meta : {
meta : { upload_by_username : self.client.user.username,
upload_by_username : self.client.user.username, upload_by_user_id : self.client.user.userId,
upload_by_user_id : self.client.user.userId, byte_size : fileSize,
byte_size : stats.size, session_temp_dl : 1, // download is valid until session is over
session_temp_dl : 1, // download is valid until session is over }
} });
});
newEntry.desc = 'File List Export'; newEntry.desc = 'File List Export';
newEntry.persist(err => { newEntry.persist(err => {
if(!err) { if(!err) {
// queue it! // queue it!
const dlQueue = new DownloadQueue(self.client); const dlQueue = new DownloadQueue(self.client);
dlQueue.add(newEntry); dlQueue.add(newEntry);
// clean up after ourselves when the session ends // clean up after ourselves when the session ends
const thisClientId = self.client.session.id; const thisClientId = self.client.session.id;
Events.once('codes.l33t.enigma.system.disconnected', evt => { // :TODO: Make a enum for system events/etc. Events.once('codes.l33t.enigma.system.disconnected', evt => { // :TODO: Make a enum for system events/etc.
if(thisClientId === _.get(evt, 'client.session.id')) { if(thisClientId === _.get(evt, 'client.session.id')) {
FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => {
if(err) { if(err) {
Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' );
} else { } else {
Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' );
} }
}); });
} }
}); });
} }
return callback(err); return callback(err);
});
}); });
}, },
function done(callback) { function done(callback) {
@ -340,4 +362,39 @@ exports.getModule = class FileBaseListExport extends MenuModule {
} }
); );
} }
getSizeAndCompressIfMeetsSizeThreshold(filePath, cb) {
fse.stat(filePath, (err, stats) => {
if(err) {
return cb(err);
}
if(stats.size < this.config.compressThreshold) {
// small enough, keep orig
return cb(null, filePath, stats.size);
}
const zipFilePath = `${filePath}.zip`;
const zipFile = new yazl.ZipFile();
zipFile.addFile(filePath, paths.basename(filePath));
zipFile.end( () => {
const outZipFile = fs.createWriteStream(zipFilePath);
zipFile.outputStream.pipe(outZipFile);
zipFile.outputStream.on('finish', () => {
// delete the original
fse.unlink(filePath, err => {
if(err) {
return cb(err);
}
// finally stat the new output
fse.stat(zipFilePath, (err, stats) => {
return cb(err, zipFilePath, stats ? stats.size : 0);
});
});
});
});
});
}
}; };