I have a very large multi section script with a LOT of loops and some recursion in it. When I run it on a Very Large dataset, the script will simply stop running. It stops with a 0 exit code. It VERY clearly does not actually finish running…it just…stops.
asyncLib.waterfall([ getPronghornToken, saveSchedulers, saveServices, populateServRefs, saveServiceGroups, saveNetworks, populateNetRefs, //never actually gets out of this function. Just exits with code 0 saveNetworkGroups, saveRuleGroups, fetchRuleGroupIds, populateRules, saveRules, getPolicyId, linkRuleGroup ], function (err, result) { if (err){ console.error("Something bad happened. Please try again"); process.exit(1); } console.log("done"); });
What I’m looking for: Why would a script just stop mid loop and exit with a 0 code?
Note: Alternate code.
getPronghornToken((err, token) => { if(err) { console.log("Error occured getPronghornToken"); throw err; } saveSchedulers(token, (err, token) => { if(err) { console.log("Error occured saveSchedulers"); throw err; } saveServices(token, (err, token) => { if(err) { console.log("Error occured saveServices"); throw err; } populateServRefs(token, (err, token) => { if(err) { console.log("Error occured populateServRefs"); throw err; } saveServiceGroups(token, (err, token) => { if(err) { console.log("Error occured saveServiceGroups"); throw err; } saveNetworks(token, (err, token) => { if(err) { console.log("Error occured saveNetworks"); throw err; } populateNetRefs(token, (err, token) => { if(err) { console.log("Error occured populateNetRefs"); throw err; } saveNetworkGroups(token, (err, token) => { if(err) { console.log("Error occured saveNetworkGroups"); throw err; } saveRuleGroups(token, (err, token) => { if(err) { console.log("Error occured saveRuleGroups"); throw err; } fetchRuleGroupIds(token, (err, token) => { if(err) { console.log("Error occured fetchRuleGroupIds"); throw err; } populateRules(token, (err, token) => { if(err) { console.log("Error occured populateRules"); throw err; } saveRules(token, (err, token) => { if(err) { console.log("Error occured saveRules"); throw err; } getPolicyId(token, (err, token) => { if(err) { console.log("Error occured getPolicyId"); throw err; } linkRuleGroup(token, (err, token) => { if(err) { console.log("Error occured linkRuleGroup"); throw err; } console.log("Successfully installed all files"); }); }); }); }); }); }); }); }); }); }); }); }); }); });
No errors thrown. Does NOT print out the innermost message. Callback pattern verified.
Last Function running looks like this:
async function populateNetRefs(token, callback) { //let newNetRefs = []; for(let index = 0; index < networkGroups.length; index++) { if (index >= networkGroups.length) { console.log("Net Refs Finished") return callback(null, token); } let networkGroup = networkGroups[index]; try { console.log(`fetching network number: ${index+1} / ${networkGroups.length}`); let newNetRefs = await fetchNetId(token, networkGroup._netRefs); networkGroup._netRefs = newNetRefs; } catch (err) { console.log(`An error occurrent fetching the network id for index ${index+1} / ${networkGroups.length}: ${err}`); } } }
The Inner Function:
function fetchNetId(token, _netRefs) { let fetchFinished = 0; let newNetRefs = []; let errCount = 1; console.log("ZZ Fetchid Start ZZ"); return new Promise((resolve, reject) => { _netRefs.forEach(function(_netRef) { let options = { //Required to be hidden }; let req = https.request(options, (res) => { let reply = []; res.setEncoding('utf8'); res.on('data', (chunk) => { console.log("YY GET DATA CHUNK YY"); reply.push(chunk); }); res.on('end', () => { fetchFinished++; console.log("Reply is : " + reply.join()); //There is some logic in this spot. Not for you. console.log("fetchFinished is: " + fetchFinished + ", size is: " + _netRefs.length); if (fetchFinished === _netRefs.length) { resolve(newNetRefs); } }); }); req.on('error', (e) => { console.error(`problem with request ${errCount++}: ${e.message}`); //reject(e); }); let body = JSON.stringify({ "options" : { "start": 0, "limit": 5, "sort": { }, "filter": { "name":{"$eq":_netRef} } } }); console.log("XX Sending Request XX"); req.write(body); req.end(); }); }); }
BEHAVIOR UPDATE – More Console Logs
Here’s the end of the console log:
fetching network number: 49 / 711 ZZ Fetchid Start ZZ XX Sending Request XX XX Sending Request XX YY GET DATA CHUNK YY Reply is : {hidden from you} TroubleShootingDias: some guid fetchFinished is: 1, size is: 2 YY GET DATA CHUNK YY Reply is : {hidden from you} TroubleShootingDias: some guid fetchFinished is: 2, size is: 2 fetch success fetching network number: 50 / 711 ZZ Fetchid Start ZZ [vagrant@vag-host1 space-parser]$
Advertisement
Answer
Actual solution to the problem: If the array is empty, then the promise never resolves. Added an empty check to the very top, before the loop.
function fetchNetId(token, _netRefs) { let fetchFinished = 0; let newNetRefs = []; let errCount = 1; console.log("ZZ Fetchid Start ZZ"); return new Promise((resolve, reject) => { if(_netRefs.length === 0) return resolve([]) // <============== _netRefs.forEach(function(_netRef) { let options = { //Required to be hidden }; let req = https.request(options, (res) => { let reply = []; res.setEncoding('utf8'); res.on('data', (chunk) => { console.log("YY GET DATA CHUNK YY"); reply.push(chunk); }); //........