问题描述
我正尝试同时下载超过100个文件.但是,当我执行下载功能时,我的Macbook在Windows中冻结(无法执行新任务),在这两种情况下(空闲网络)也都没有下载(但没有冻结)并且没有下载进度.
I am trying to download more that 100 files at the same time. But when I execute the downloading function my macbook freezes(unable to execute new tasks) in windows also no download(but doesn't freeze) and no download progress in both case(idle network).
这是我的下载模块:
var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');
router.get('/', function (req, res, next) {
links.find({dlStatus: false}, function (err, docs) {
if (err) {
console.log(err);
res.end();
} else if (!docs) {
console.log('No incomplete downloads!');
res.end();
} else {
for (var i = 0; i < docs.length; i++) {
//todo scraping
var video = youtubedl(docs[i].url, [], {cwd: __dirname});
// Will be called when the download starts.
video.on('info', function (info) {
console.log('Download started');
console.log(info);
});
video.pipe(fs.createWriteStream('./downloads/' + docs[i].id + '-' + i + '.mp4'));
video.on('complete', function complete(info) {
links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
if (err)console.log(err);
else console.log('Download completed!')
});
});
}
}
});
});
module.exports = router;
现在任何人都可以在这里帮助我吗?我正在使用此模块下载文件.
Now can anyone please help me here? I am using this module for downloading files.
推荐答案
在这种情况下,解决方案正在使用 async .
The solution is using async in this case.
以这种方式尝试....使用async.each()
Try it this way....with async.each()
var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');
var async = require('async')
router.get('/', function (req, res, next) {
links.find({dlStatus: false}, function (err, docs) {
if (err) {
console.log(err);
res.end();
} else if (!docs) {
console.log('No incomplete downloads!');
res.end();
} else {
async.each(docs,function(doc,cb){
var video = youtubedl(doc.url, [], {cwd: __dirname});
// Will be called when the download starts.
video.on('info', function (info) {
console.log('Download started');
console.log(info);
});
video.pipe(fs.createWriteStream('./downloads/' + docs.id + '-' + i + '.mp4'));
video.on('complete', function complete(info) {
links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
if (err){
console.log(err);
cb(err);
}
else {
console.log('Download completed!');
cb()
}
});
});
},function(err){
if(err)
return console.log(err);
console.log("Every thing is done,Here!!");
})
}
});
});
module.exports = router;
您也可以使用async.eachLimits()
批量处理所有内容.
And you can process every thing in batch too using async.eachLimits()
.
这篇关于Node.JS同时下载数百个文件的文章就介绍到这了,希望我们推荐的答案对大家有所帮助,也希望大家多多支持!