Myujth

Imagine for example that you want to store paginated data from an API to a database.

let db;
let pageitems = 35
var offset = 0;

dbConnect //establish connection to database 
 .then( fetch(apiLink+?offset=2) 
 .then( res => res.json())
 .then( res => {
 var total = res.count
 return collection.insertMany(res.data, ordered: false)
 // If offset is less than total, I want to increase offset and go back to the fetch-event.
 .catch( err => 
 if(err.code !== 11000)log(err)
 elselog(completed: err.result.nInserted, duplicates: 
 err.result.result.writeErrors.length);
 )
 .then(() => 
 connection.close();
 )

You could just use a regular loop:

 (async function() 
 const conn = await dbConnect;
 for(let offset = 0; true; offset++) 
 const data, count = await (await fetch(`api?page=$offset`)).json();
 // Exit if the page is empty
 if(count === 0) break;
 await collection.insertMany(data, ordered: false );
 
 )();

To speed that up you could execute multiple requests in parallel:

 const chunkSize = 10; // 10 in parallel
 for(let offset = 0; offset < chunkSize; offset++) 
 (async function() 
 const conn = await dbConnect;
 for(let offset2 = 0; true; offset2 += chunkSize) 
 const data, count = await (await fetch(`api?page=$offset + offset2`)).json();
 // Exit if the page is empty
 if(count === 0) break;
 await collection.insertMany(data, ordered: false );
 
 )();
 

Basically, you will want to wrap your fetch and insert into a function that you will call many times. See the below as an example to illustrate my point...

let db;
let pageitems = 35
var offset = 0;


var db = dbConnect() //establish connection to database 

function fetch_and_insert(offset) 
 db
 .then(fetch(apiLink + "?" + offset))
 .then(res => res.json())
 .then(res => 
 var total = res.count
 collection.insertMany(res.data, ordered: false )
 .catch(err => 
 if (err.code !== 11000) log(err) 
 else 
 log(
 completed: err.result.nInserted, duplicates: err.result.result.writeErrors.length
 );
 
 )
 if (offset < total) return fetch_and_insert(offset + pageitems)
 return null;
 )


fetch_and_insert(offset)
.then(() => 
 connection.close();
)