lulz
function zdownload(f,file,cb){
var done = false;
var ext = file.split('.').pop();
try {
f.get(file,false,function(err,stream){
if(err || stream === u){
done = true;
console.logPush( 'error on download'.red, file, err.message);
return cb(err,null);
}
if(ext === 'zip'){
var out = {};
stream.pipe(unzip.Parse())
.on('entry', function(entry){
var name = entry.path;
if(entry.type !== 'File') return entry.autodrain();
getStream(entry,function(err,data){
if(done) return;
if(err)
return console.logPush('unzip error'.red,file+'/'+name, err.message);
out[name] = data;
});
}).on('close',function(){
if(done) return; done = true;
return cb(null,out);
});
} else {
getStream(stream,function(err,data){
if(done) return; done = true;
var out = {}; out[file] = data;
return cb(err,out);
});
}
});
} catch(e){
if(done) return; done = true;
console.logPush('unhandled ftp download error'.red, file, e.message);
return cb(err,null);
}
}
function ftpGet(nfo,cb){
var t = this;
var startTime = Date.now();
var keyprefix = 'tourinfrance:'+t.nfo.name+':ftpFiles:';
createFtp(nfo.server,function(err,f){ // create main ftp
f.list(nfo.folder,false,function(err,list){ // get files list
if(err){
console.logPush('error',err);
return f.end();
}
console.log('listing done'.yellow,list.length,'entry');
if(nfo.match){
_.remove(list,function(file){
return ( file.type !== '-' || !mm([file.name],nfo.match).length );
});
console.log(list.length.toString().magenta,'entry keep after filtering',nfo.match);
}
var toDl = [], files = {}; // array of files to download
// check cache for each files
async.each(list,function(e,cb){
if(e.type !== '-') return setTimeout(cb,1); // not a file
//var ext = e.name.split('.').pop();
//if(nfo.validExt !== u && _.indexOf(nfo.validExt,ext) === -1) return setTimeout(cb,1); // not valid files
if( useCache !== true ){ toDl.push( e ); return setTimeout(cb,1); } // not use cache
var key = keyprefix+e.name;
t.db.hgetall(key,function(err,file){
if(err || file === null || file.size != e.size){ toDl.push( e ); return setTimeout(cb,1); }
var data = JSON.parse(file.data);
_.forIn(data,function(data,key){
files[key] = data;
});
cb();
});
},function(err){ // list of files to dl is generated
function allDlDone(){
var totalBytes = 0;
_.forIn(files,function(data,name){
totalBytes += data.length;
});
console.log('total files number'.yellow,Object.keys(files).length, '('+parseInt(totalBytes/1024,10)+'kb)');
cb(null,files);
}
if(!toDl.length) return allDlDone();
var maxFtp = nfo.parallelLimit === u ? 1 : nfo.parallelLimit;
if(toDl.length < maxFtp) maxFtp=1;
console.logPush('download',toDl.length,'files with',maxFtp,'// ftp');
async.timesLimit(maxFtp-1, 10, function(n, next) {
createFtp(nfo.server,next);
}, function(err, ftps){ // all additionnal ftp connected
ftps.push(f);
var ftpFree = []; _.each(ftps,function(f,n){ ftpFree.push(n); });
function gen_download(e,file){
return function(cb){
var ftpn = ftpFree.shift();
function createDl(){
return async.timeout(function(cb){ zdownload(ftps[ftpn],file,cb); }, nfo.timeout);
}
var zdl = function(err,data){
if(err){
console.logPush('download'.red,file.yellow,'need retry'.red,'ftp#'+ftpn,err.code,err.message);
ftps[ftpn].end(); // shutdown old client
createFtp(nfo.server,function(err,f){
ftps[ftpn] = f;
ftpFree.push(ftpn);
ftpn = ftpFree.shift();
createDl()(zdl);// dl(zdl);
});
} else return zcb(null,data);
}
var zcb = function(err,data){
ftpFree.push(ftpn);
if(err){
console.logPush('error on download'.red,file,err.message);
}
_.forIn(data,function(data,key){ files[key] = data; });
var key = keyprefix+e.name;
t.db.multi().hmset(key,{ data:JSON.stringify(data), size:e.size }).expire(key,60*60*24).exec(function(err,repl){
cb(null);
});
}
createDl()(zdl);
};
}
var jobs = []; _.each(toDl,function(e,n){ jobs.push(
function(cb){
gen_download(e,nfo.folder+e.name,n)(function(){
console.logLast('downloaded'.green,e.name.yellow,n+'/'+toDl.length);
cb();
})
}
); });
async.parallelLimit(jobs,maxFtp,function(){ // got all files
var time = (Date.now() - startTime)/1000;
async.each(ftps,function(f,cb){ // shutdown all ftp client
f.on('close',function(){ cb(); }); f.end();
}, function(){ // show all files
console.logPush(toDl.length.toString().green,'ftp downloads done'.green, time+'s');
//cb(null,files);
allDlDone();
});
});
});
});
});
});
}
)
