Looping in RSelenium and Scraping - for-loop

Question

I'm trying to scrape data from website using RSelenium. I am able to navigate through drop downs individually but when I run them in loop I get error.
Also after selecting all the values in the drop down I want to store the name of the facility and contact details in a table. Which I'm not able to do so far.
rm(list=ls())
setwd("D:\\work_codes\\kvk\\data")
getwd()
library(RSelenium)
library(rvest)
library(XML)
library(RCurl)
library(magrittr)
library(stringr)
rd<-rsDriver()
remDr<-rd[["client"]]
remDr$navigate("https://kvk.icar.gov.in/facilities_list.aspx")
remDr$refresh()
stateEle<-remDr$findElement("id", "ContentPlaceHolder1_ddlState")
states<-stateEle$getElementText()[[1]] %>% strsplit(., '\\n') %>% unlist %>% setdiff(., ' --Select--')
states<-str_trim(states, 'left')
stateEle$clickElement()
for (i in 1:length(states)) {
remDr$refresh()
stateEle$clickElement()
stateEle$sendKeysToElement(list(states[i]))
stateEle$clickElement()
districts<-NULL
distEle<-remDr$findElement("id", "ContentPlaceHolder1_ddlDistrict")
districts<-distEle$getElementText()[[1]] %>% strsplit(., '\\n') %>% unlist %>% setdiff(., ' --Select--')
districts<-str_trim(districts, 'left')
for (j in 1:length(districts)) {
distEle$clickElement()
distEle$sendKeysToElement(list(districts[j]))
distEle$clickElement()
kvk<-NULL
kvkEle<-remDr$findElement("id", "ContentPlaceHolder1_ddlKvk")
kvk<-kvkEle$getElementText()[[1]] %>% strsplit(., '\\n') %>% unlist %>% setdiff(., ' --Select--')
kvk<-str_trim(kvk, 'left')
for (k in 1:length(kvk)) {
kvkEle$clickElement()
kvkEle$sendKeysToElement(list(kvk[[1]]))
kvkEle$clickElement()
submitEle<-remDr$findElement("id", "ContentPlaceHolder1_btnSubmit")
submitEle$clickElement()
doc<-remDr$findElement('id', 'ContentPlaceHolder1_rptfacility_f_name_1')
doc$getElementText()
doc$clickElement()
remDr$findElement('class name','Contact details:')
}
}
}

Answer 1

library(rvest)
url<-"https://kvk.icar.gov.in/facilities_list.aspx"
page<-html_session(url)
form<-html_form(page)[[1]]
states<-html_nodes(page,css="#ContentPlaceHolder1_ddlState > option") %>% html_attr("value")
states<-states[-1]
states_name<-html_nodes(page,css="#ContentPlaceHolder1_ddlState > option") %>% html_text()
states_name<-states_name[-1]
final_df<-0
#### STATES LOOP ####
for(i in 1:length(states)){
filled_form<-set_values(form,
"ctl00$ContentPlaceHolder1$ddlState"=states[i])
page1<-submit_form(page,filled_form)
district<-html_nodes(page1,css="#ContentPlaceHolder1_ddlDistrict > option") %>% html_attr("value")
district<-district[-1]
district_name<-html_nodes(page1,css="#ContentPlaceHolder1_ddlDistrict > option") %>% html_text()
district_name<-district_name[-1]
#### DISTRICT LOOP ####
for(j in 1:length(district)){
filled_form1<-set_values(html_form(page1)[[1]],
"ctl00$ContentPlaceHolder1$ddlState"=states[i],
"ctl00$ContentPlaceHolder1$ddlDistrict"=district[j])
page2<-submit_form(page1,filled_form1)
kvk<-html_nodes(page2,css="#ContentPlaceHolder1_ddlKvk > option") %>% html_attr("value")
kvk<-kvk[-1]
kvk_name<-html_nodes(page2,css="#ContentPlaceHolder1_ddlKvk > option") %>% html_text()
kvk_name<-kvk_name[-1]
#### KVK LOOP ####
for(k in 1:length(kvk)){
filled_form2<-set_values(html_form(page2)[[1]],
"ctl00$ContentPlaceHolder1$ddlState"=states[i],
"ctl00$ContentPlaceHolder1$ddlDistrict"=district[j],
"ctl00$ContentPlaceHolder1$ddlKvk"=kvk[k])
page3<-submit_form(page2,filled_form2)
contact_text<-gsub("[\r\n]","",html_nodes(page3,css=".panel-body") %>% html_text())
if(length(contact_text) == 0){contact_text=""}
df<-data.frame(cbind(states_name[i],district_name[j],kvk[k],contact_text))
names(df)<-c("STATE","DISTRICT","KVK","CONTACT_TEXT")
final_df[i*j*k] = list(df)
### WAITTIME TO AVOID HTTP 500 error - So the server is not overloaded
sleep(5)
}
}
}
output_df<-data.table::rbindlist(final_df,fill=TRUE)
# After this perform some string operations to extract the exact information required from the CONTACT_TEXT variable
The above answer does not use any RSelenium package and I think this is more trustworthy than RSelenium.