Hi,
I want to scrape data from Registrations
When I search a major keyword it returns with limit exceed.
There is javascript of search in the HTML:
Visualforce.remoting.timeout = 150000; // Set timeout at page level
var $ = jQuery.noConflict();
var allIds = [];
var regIds = [];
var regStart=0;
var filters=[];
var searchBy;
var startDate;
var expirationDate;
var county;
var city;
var fromDOM=false;
var displayCount=0;
var myList = [];
var TotalPages=0;
var CurrentPage=0;
var maxRegistrations = 10000;
cookieString = 'Search_MyList';
$(document).ready(function() {
fromDOM=true;
//filter change method
//registerOnFilterChange();
//get all Ids
getRegKeys();
//register scroll magic
//registerScrollMagic();
//cookies
myList = Cookies.get(cookieString);
if (myList) {
myList = JSON.parse(myList);
} else {
myList = [];
}
updateMyListCount();
fromDOM=false;
$(".slds-input").keypress(function(e) {
if (e.keyCode === 13) { // If Enter key pressed
SearchRegs();
}
});
});
function viewDetails(regId){
url="/ContractorSearch/registrationSearchDetails?id="+regId;
window.open(url,"_blank");
}
window.onscroll = function() {scrollFunction()};
function scrollFunction() {
if (document.body.scrollTop > 500 || document.documentElement.scrollTop > 500) {
document.getElementById("myBtn").style.display = "block";
} else {
document.getElementById("myBtn").style.display = "none";
}
}
function scrollToTop(){
$("html, body").animate({ scrollTop: 0 }, "slow");
return false;
//document.body.scrollTop = document.documentElement.scrollTop = 0;
}
function registerOnFilterChange(){
$("#totalResultCount").empty();
$("#showingPages").empty();
//$(".filter-group :input").on('change', function(){
UpdateURL();
getRegKeys();
//});
}
function registerScrollMagic(){
var smController = new ScrollMagic.Controller();
var smScene = new ScrollMagic.Scene({triggerElement: "#footer", triggerHook: "onEnter"})
.addTo(smController)
.on("enter", function (e) {
if (regStart >= allIds.length) return;
$("#spinner").show();
getRegListNames();
});
}
function reset(){
$("#SearchByInput").val("");
$("#StartDate").val("");
$("#ExpirationDate").val("");
$("#County").val("");
$("#City").val("");
$(".filter-group :input").each(function(){
this.checked=false;
});
$("#regulations").empty();
$("#totalResultCount").empty();
$("#showingPages").empty();
SearchRegs();
}
function constructURLFilterParam(){
var filtersString='';
populateFilters();
if(filters.length>0){
for(k=0; k < filters.length; k++){
filtersString+=filters[k]+';';
}
filtersString = filtersString.slice(0, -1);
}
return filtersString;
}
function populateFilters(){
filters=[];
$(".filter-group :input").each(function(){
if(this.checked){
filters.push(this.name);
}
});
}
function UpdateURL(){
var searchObject={
inputSearch:$("#SearchByInput").val(),
startDate:$("#StartDate").val(),
expirationDate:$("#ExpirationDate").val(),
county:$("#County").val(),
city:$("#City").val(),
filters:constructURLFilterParam()
};
var newURL = 'ContractorSearch?'+ Object.keys(searchObject).filter(function(k) { return searchObject[k]; }).map(function(k) { return k + '=' + searchObject[k]; }).join('&');
history.replaceState('test', '', newURL);
}
$.getQueryParameters = function(str) {
return (str || document.location.search).replace(/(^\?)/,'').split("&").map(function(n){return n = n.split("="),this[n[0]] = n[1],this}.bind({}))[0];
}
function readURLParams(){
var params = $.getQueryParameters();
$.extend(params,{
foo:4
});
if (typeof params.inputSearch != 'undefined') {
searchBy = params.inputSearch
}else{
searchBy = null;
}
//search for startDate
if (typeof params.startDate != 'undefined') {
startDate = params.startDate
}else{
startDate = null;
}
//search for expiration
if (typeof params.expirationDate != 'undefined') {
expirationDate = params.expirationDate
}else{
expirationDate = null;
}
//search for location
if (typeof params.county != 'undefined') {
county = params.county
}else{
county = null;
}
//search for city
if (typeof params.city != 'undefined') {
city = params.city;
}else{
city = null;
}
var filterParam=params.filter;
if(filterParam){
filters=filterParam.split(";");
}
if(fromDOM){
if(searchBy){
$("#SearchByInput").val(searchBy);
}
if(startDate){
$("#StartDate").val(startDate);
}
if(expirationDate){
$("#ExpirationDate").val(expirationDate);
}
if(county){
$("#County").val(county);
}
if(city){
$("#City").val(city);
}
//check the filter checkboxes
if(filterParam){
filtersArray=filterParam.split(";");
for(f=0; f < filtersArray.length; f++){
$(".filter-group :input").each(function(){
if(this.name==filtersArray[f]){
this.checked=true;
}
});
}
}
}
}
function SearchRegs(){
/* $(".filter-group :input").each(function(){
this.checked=false;
});*/
$("#totalResultCount").empty();
$("#showingPages").empty();
UpdateURL();
getRegKeys();
}
function getRegKeys(){
allIds = [];
regIds = [];
regStart=0;
//var searchBy=$("#SearchByInput").val();
readURLParams();
//populateFilters();
if(searchBy || startDate || expirationDate || county || city){
$("#spinner").show();
$("#regulations").empty();
Visualforce.remoting.Manager.invokeAction(
'RegistrationSearchNewController.getKeys',
searchBy,startDate,expirationDate,county,city,filters,
regCallback,
{escape: false}
);
}
}
function handlePaginationButtons(){
if(currentPage==TotalPages){
$("#next").attr("disabled",true);
}else{
$("#next").attr("disabled",false);
}
if(currentPage==1){
$("#previous").attr("disabled",true);
}else{
$("#previous").attr("disabled",false);
}
$("#showingPages").html("Showing Page "+currentPage+" of "+TotalPages);
}
var regCallback=function(regs, event){
if (regs != null){
console.log('response is '+regs);
regIds = regs[0];…
Can you please check if we can turn off the limit?
I have another scraper that scrapes the HTML which is open in the console. if I get able to show all the companies list or the companies with a keyword that I searched, then I can get all data without refreshing the page.
Tell me if you have any idea about this.