node script just stops

I have a very large multi section script with a LOT of loops and some recursion in it. When I run it on a Very Large dataset, the script will simply stop running. It stops with a 0 exit code. It VERY clearly does not actually finish running…it just…stops.

asyncLib.waterfall([
   getPronghornToken,
   saveSchedulers,
   saveServices,
   populateServRefs,
   saveServiceGroups,
   saveNetworks,
   populateNetRefs,      //never actually gets out of this function.  Just exits with code 0
   saveNetworkGroups,
   saveRuleGroups,
   fetchRuleGroupIds,
   populateRules,
   saveRules,
   getPolicyId,
   linkRuleGroup
], function (err, result) {
   if (err){
       console.error("Something bad happened. Please try again");
       process.exit(1);
   }
   console.log("done");
});

What I’m looking for: Why would a script just stop mid loop and exit with a 0 code?

Note: Alternate code.

getPronghornToken((err, token) => {
if(err) {
    console.log("Error occured getPronghornToken");
    throw err;
}
   saveSchedulers(token, (err, token) => {
    if(err) {
        console.log("Error occured saveSchedulers");
        throw err;
    }
    saveServices(token, (err, token) => {
        if(err) {
            console.log("Error occured saveServices");
            throw err;
        }
        populateServRefs(token, (err, token) => {
            if(err) {
                console.log("Error occured populateServRefs");
                throw err;
            }
            saveServiceGroups(token, (err, token) => {
                if(err) {
                    console.log("Error occured saveServiceGroups");
                    throw err;
                }
                saveNetworks(token, (err, token) => {
                    if(err) {
                        console.log("Error occured saveNetworks");
                        throw err;
                    }
                    populateNetRefs(token, (err, token) => {
                        if(err) {
                            console.log("Error occured populateNetRefs");
                            throw err;
                        }
                        saveNetworkGroups(token, (err, token) => {
                            if(err) {
                                console.log("Error occured saveNetworkGroups");
                                throw err;
                            }
                            saveRuleGroups(token, (err, token) => {
                                if(err) {
                                    console.log("Error occured saveRuleGroups");
                                    throw err;
                                }
                                fetchRuleGroupIds(token, (err, token) => {
                                    if(err) {
                                        console.log("Error occured fetchRuleGroupIds");
                                        throw err;
                                    }
                                    populateRules(token, (err, token) => {
                                        if(err) {
                                            console.log("Error occured populateRules");
                                            throw err;
                                        }
                                        saveRules(token, (err, token) => {
                                            if(err) {
                                                console.log("Error occured saveRules");
                                                throw err;
                                            }
                                            getPolicyId(token, (err, token) => {
                                                if(err) {
                                                    console.log("Error occured getPolicyId");
                                                    throw err;
                                                }
                                                linkRuleGroup(token, (err, token) => {
                                                    if(err) {
                                                        console.log("Error occured linkRuleGroup");
                                                        throw err;
                                                    }

                                                    console.log("Successfully installed all files");
                                                });
                                            });
                                        });
                                    });
                                });
                            });
                        });
                    });
                });
            });
        });
    });
   });
});

No errors thrown. Does NOT print out the innermost message. Callback pattern verified.

Last Function running looks like this:

async function populateNetRefs(token, callback) {
//let newNetRefs = [];
for(let index = 0; index < networkGroups.length; index++) {
    if (index >= networkGroups.length) {
        console.log("Net Refs Finished")
        return callback(null, token);
    }
    let networkGroup = networkGroups[index];
    try {
        console.log(`fetching network number: ${index+1} / ${networkGroups.length}`);
        let newNetRefs = await fetchNetId(token, networkGroup._netRefs);

        networkGroup._netRefs = newNetRefs;
    } catch (err) {
        console.log(`An error occurrent fetching the network id for index ${index+1} / ${networkGroups.length}: ${err}`);
    }
}
}

The Inner Function:

function fetchNetId(token, _netRefs) {
let fetchFinished = 0;
let newNetRefs = [];
let errCount = 1;
console.log("ZZ Fetchid Start ZZ");
return new Promise((resolve, reject) => {
    _netRefs.forEach(function(_netRef) {
        let options = {
            //Required to be hidden
        };

        let req = https.request(options, (res) => {
            let reply = [];
            res.setEncoding('utf8');
            res.on('data', (chunk) => {
                console.log("YY GET DATA CHUNK YY");
                reply.push(chunk);
            });
            res.on('end', () => {
                fetchFinished++;
                console.log("Reply is : " + reply.join());
                //There is some logic in this spot.  Not for you.
                console.log("fetchFinished is: " + fetchFinished + ", size is: " + _netRefs.length);
                if (fetchFinished === _netRefs.length) {
                    resolve(newNetRefs);
                }
            });
        });
        req.on('error', (e) => {
            console.error(`problem with request ${errCount++}: ${e.message}`);
            //reject(e);
        });

        let body = JSON.stringify({
            "options" : {
                "start": 0,
                "limit": 5,
                "sort": {

                },
                "filter": {
                    "name":{"$eq":_netRef}
                }
            }
        });
        console.log("XX Sending Request XX");
        req.write(body);
        req.end();
    });
});
}

BEHAVIOR UPDATE – More Console Logs

Here’s the end of the console log:

fetching network number: 49 / 711
ZZ Fetchid Start ZZ
XX Sending Request XX
XX Sending Request XX
YY GET DATA CHUNK YY
Reply is : {hidden from you}
TroubleShootingDias: some guid
fetchFinished is: 1, size is: 2
YY GET DATA CHUNK YY
Reply is : {hidden from you}
TroubleShootingDias: some guid
fetchFinished is: 2, size is: 2
fetch success
fetching network number: 50 / 711
ZZ Fetchid Start ZZ
[vagrant@vag-host1 space-parser]$

Answer

Actual solution to the problem: If the array is empty, then the promise never resolves. Added an empty check to the very top, before the loop.

function fetchNetId(token, _netRefs) {
let fetchFinished = 0;
let newNetRefs = [];
let errCount = 1;
console.log("ZZ Fetchid Start ZZ");
return new Promise((resolve, reject) => {

if(_netRefs.length === 0) return resolve([]) // <==============

_netRefs.forEach(function(_netRef) {
    let options = {
        //Required to be hidden
    };

    let req = https.request(options, (res) => {
        let reply = [];
        res.setEncoding('utf8');
        res.on('data', (chunk) => {
            console.log("YY GET DATA CHUNK YY");
            reply.push(chunk);
        });
//........