Scrape sources in bulk

by Marcel Landeweerd

HTML

<textarea id="domain_list" style="width:300px; height:200px"></textarea>
<button onclick="createProfilesFromList(); return false" type="button">Create Profiles</button>

<iframe width="300" height="200" id="loadProfile"></iframe>

JavaScript

async function createProfilesFromList() {
var textArea = document.getElementById("domain_list");
var arrayOfDomains = textArea.value.split("\n");
var arrayLength = arrayOfDomains.length;
for (var i = 0; i < arrayLength; i++) {
		let domain = (new URL(getValidUrl(arrayOfDomains[i])));
    //domain = domain.hostname.replace('www.','');
    alert("https://trustdb.webwinkelkeur.nl/scrape?url="+domain);
    document.getElementById('loadProfile').src = "https://trustdb.webwinkelkeur.nl/scrape?url="+domain;
    await sleep(3000);
	}
}

function getValidUrl(url = "") {
    let newUrl = window.decodeURIComponent(url);
    newUrl = newUrl.trim().replace(/\s/g, "");

    if(/^(:\/\/)/.test(newUrl)){
        return `http${newUrl}`;
    }
    if(!/^(f|ht)tps?:\/\//i.test(newUrl)){
        return `http://${newUrl}`;
    }

    return newUrl;
}

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
  }