I'm scraping three drop down menus and links after drop down selections. I need to use greasemonkey/tapermonkey script to get info of URL what I need. So if you really want to test this yourself you need to install this script also. Everything goes fine but somewhere before middle of Audi model list scraping just stops and no data.
Tampermonkey/greasemonkey script what is needed:
// ==UserScript==
// @name KW URL overview
// @namespace http://tampermonkey.net/
// @version 1.0
// @description Update a div element on a specific page every 500ms
// @author Your Name
// @match https://www.kwsuspensions.net/*
// @grant none
// ==/UserScript==
(function() {
'use strict';
const divClass = 'header-dropdowns' // header-dropdowns
const checkInterval = 100; // in milliseconds
const idToReplace = 'url-content'; // ID for the element to replace
function updateDiv() {
let targetDiv = document.querySelector(`.${divClass}`);
if (targetDiv) {
let existingDiv = document.getElementById(idToReplace);
if (existingDiv) {
existingDiv.textContent = window.location.href;
} else {
let newContent = document.createElement('div');
newContent.id = idToReplace;
newContent.textContent = window.location.href;
targetDiv.insertBefore(newContent, targetDiv.firstChild);
}
}
}
setInterval(updateDiv, checkInterval);
})();
Url: Shop - Order products online
Sitemap:
{"_id":"KW_suspensions_merkit_tuoteoverview","startUrl":["https://www.kwsuspensions.net/shop"],"selectors":[{"clickActionType":"real","clickElementSelector":"#car-selector-collapse .makesList li:nth-of-type(7)","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":1200,"discardInitialElements":"discard","id":"merkki","multiple":false,"parentSelectors":["_root"],"selector":"div.page-wrapper","type":"SelectorElementClick"},{"clickActionType":"real","clickElementSelector":".collapse .modelsList span","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":1200,"discardInitialElements":"discard","id":"malli","multiple":true,"parentSelectors":["merkki"],"selector":"_parent_","type":"SelectorElementClick"},{"clickActionType":"real","clickElementSelector":".collapse .carsList span","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":1500,"discardInitialElements":"discard","id":"moottori","multiple":true,"parentSelectors":["malli"],"selector":"_parent_","type":"SelectorElementClick"},{"id":"merkkiteksti","multiple":false,"parentSelectors":["tuoteryhmä"],"regex":"","selector":".collapse span[data-bind='text: carTextArray().make']","type":"SelectorText"},{"id":"malliteksti","multiple":false,"parentSelectors":["tuoteryhmä"],"regex":"","selector":".collapse span[data-bind='text: carTextArray().model']","type":"SelectorText"},{"id":"moottoriteksti","multiple":false,"parentSelectors":["tuoteryhmä"],"regex":"","selector":".collapse .selectedCar__text--motor span","type":"SelectorText"},{"clickActionType":"real","clickElementSelector":".multi-collapse-SuspensionTechnology li:nth-of-type(1):contains('Show'), .multi-collapse-SuspensionTechnology li:nth-of-type(2):contains('Show'), .multi-collapse-WheelSpacers li:nth-of-type(1):contains('Show')","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":1200,"discardInitialElements":"do-not-discard","id":"tuoteryhmä","multiple":true,"parentSelectors":["moottori"],"selector":"_parent_","type":"SelectorElementClick"},{"id":"tuoteotsikko","multiple":true,"parentSelectors":["tuoteryhmä"],"regex":"","selector":".product-title div","type":"SelectorText"},{"id":"url","multiple":false,"parentSelectors":["tuoteryhmä"],"regex":"","selector":"div#url-content","type":"SelectorText"},{"clickActionType":"real","clickElementSelector":"a[data-bind='click: onBreadcrumbsProductOverviewCategoryOverviewClick']","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":800,"discardInitialElements":"do-not-discard","id":"takaisin-overview","multiple":false,"parentSelectors":["tuoteryhmä"],"selector":"_parent_","type":"SelectorElementClick"}]}