.Rhistory

}
list_files <- list.files(folder, pattern = "0.*")
list_files_path <-  list.files(folder, pattern = "0.*", full.names = TRUE)
length(list_files)
# delete non-existing articles
files_size <- sapply(list_files_path, file.size)
table(files_size) %>% sort()
delete_files <- list_files_path[files_size == 24265]
sapply(delete_files, file.remove)
list_files_path <-  list.files(folder, pattern = "0.*", full.names = TRUE) # update list of files
# import pages and extract content
authors <- character()
title <- character()
statistics <- character()
numViews <- numeric()
datePublish <- character()
for (i in 1:length(list_files_path)) {
html_out <- read_html(list_files_path[i])
table_out <- html_table(html_out, fill = TRUE)[[6]]
authors[i] <- table_out[1,2]
title[i] <- table_out[2,2]
statistics[i] <- table_out[4,2]
numViews[i] <- statistics[i] %>% str_extract("[[:digit:]]+") %>% as.numeric()
datePublish[i] <- statistics[i] %>% str_extract("[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}.$") %>% str_replace("\\.", "")
}
# construct data frame
dat <- data.frame(authors = authors, title = title, numViews = numViews, datePublish = datePublish)
head(dat)
# download statistics
dattop <- dat[order(dat$numViews, decreasing = TRUE),]
dattop[1:10,]
summary(dat$numViews)
plot(density(dat$numViews, from = 0, ), yaxt="n", ylab="", xlab="Number of views", main="Distribution of article page views in JSTATSOFT")
getwd()
setwd("/Users/simon/GitHub/Web-scraping-with-R-Extended-Edition/")
url <- "http://www.iea.org/policiesandmeasures/renewableenergy/"
browseURL(url)
content <- read_html(url)
html_content(/Users/simon/GitHub/Web-scraping-with-R-Extended-Edition/)
html_content(content)
html_structure(content)
checkForServer()
vignette("RSelenium-docker", package = "RSelenium")
vignette("RSelenium-basics", package = "RSelenium")
startServer()
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr$navigate(url)
remDr$open()
remDr$navigate(url)
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "chrome")
remDr$open()
?remoteDriver
remDr <- remoteDriver$new()
remDr$open()
forecast <- read_xml("http://export.arxiv.org/api/query?search_query=all:forecast")
xml_ns(forecast) # inspect namespaces
authors <- xml_find_all(forecast, "//d1:author", ns = xml_ns(forecast))
authors %>% xml_text()
library(aRxiv)
browseURL("http://ropensci.org/tutorials/arxiv_tutorial.html")
ls("package:aRxiv")
arxiv_df <- arxiv_search(query = "forecast AND submittedDate:[2016 TO 2017]", limit = 500, output_format = "data.frame")
View(arxiv_df)
arxiv_count('au:"Gary King"')
query_terms
arxiv_count('abs:"political" AND submittedDate:[2016 TO 2017]')
polsci_articles <- arxiv_search('abs:"political" AND submittedDate:[2016 TO 2017]', limit = 500)
library(pageviews)
trump_views <- article_pageviews(project = "en.wikipedia", article = "Donald Trump", user_type = "user", start = "2016010100", end = "20160729")
head(trump_views)
clinton_views <- article_pageviews(project = "en.wikipedia", article = "Hillary Clinton", user_type = "user", start = "2016010100", end = "20160729")
trump_views <- article_pageviews(project = "en.wikipedia", article = "Donald Trump", user_type = "user", start = "2016010100", end = "20161011")
head(trump_views)
clinton_views <- article_pageviews(project = "en.wikipedia", article = "Hillary Clinton", user_type = "user", start = "2016010100", end = "20161011")
plot(ymd_h(trump_views$timestamp), trump_views$views, col = "red", type = "l")
lines(ymd_h(clinton_views$timestamp), clinton_views$views, col = "blue")
title <- "Groundhog Day" %>% URLencode()
endpoint <- "http://www.omdbapi.com/?"
url <- paste0(endpoint, "t=", title, "&tomatoes=true")
omdb_res = GET(url)
res_list <- content(omdb_res, as =  "parsed")
res_list %>% unlist() %>% t() %>% data.frame(stringsAsFactors = FALSE)
url <- paste0(endpoint, "s=", title)
omdb_res = GET(url)
res_list <- content(omdb_res, as = "text") %>% jsonlite::fromJSON(flatten = TRUE)
res_list$Search
apikey <- "&appid=42c7829f663f87eb05d2f12ab11f2b5d"
endpoint <- "http://api.openweathermap.org/data/2.5/find?"
city <- "Chicago,IL"
metric <- "&units=metric"
url <- paste0(endpoint, "q=", city, metric, apikey)
weather_res <- GET(url)
res_list <- content(weather_res, as =  "parsed")
res_list <- content(weather_res, as =  "text")  %>% jsonlite::fromJSON(flatten = TRUE)
res_list$list
api_key <- Sys.getenv("twitter_api_key")
api_secret <- Sys.getenv("twitter_api_secret")
setup_twitter_oauth(api_key,api_secret)
tweets <- searchTwitter(searchString = "Trump", n=25)
tweets_df <- twListToDF(tweets)
head(tweets_df)
names(tweets_df)
load("../rscraping-intro-duke-2/twitter_auth.RData")
filterStream("tweets_stream.json", track = c("Trump"), timeout = 10, oauth = twitCred)
tweets <- parseTweets("tweets_stream.json", simplify = TRUE)
names(tweets)
cat(tweets$text[1])
RSelenium::checkForServer(beta = TRUE)
source("00-course-setup.r")
wd <- getwd()
devtools::install_github("ropensci/Rselenium")
library(RSelenium)
source("00-course-setup.r")
selServ <- RSelenium::startServer(javaargs = c("-Dwebdriver.gecko.driver=\"/Users/simon/GitHub/Web-scraping-with-R-Extended-Edition/geckodriver .exe\""))
RSelenium::checkForServer(beta = TRUE)
startServer()  ## deprecated
remDr <- remoteDriver$new()
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
url <- "http://www.iea.org/policiesandmeasures/renewableenergy/"
remDr <- remoteDriver(extraCapabilities = list(marionette = TRUE))
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr$navigate(url)
remDr <- remoteDriver(extraCapabilities = list(marionette = TRUE))
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr$navigate(url)
vignette('RSelenium-basics')
RSelenium::checkForServer(beta = TRUE) ## deprecated; but gives advice on how to source/start a server
vignette("RSelenium-basics", package = "RSelenium")
vignette("RSelenium-docker", package = "RSelenium")
vignette("RSelenium-basics", package = "RSelenium")
?startServer
selServ <- RSelenium::startServer(javaargs = c("-Dwebdriver.gecko.driver=\"geckodriver .exe\""))
vignette("RSelenium-basics", package = "RSelenium")
remDr <- remoteDriver$new()
remDr <- remoteDriver(extraCapabilities = list(marionette = TRUE))
?remoteDriver
remDr <- remoteDriver(extraCapabilities = list(marionette = TRUE))
remDr$open()
remDr <- remoteDriver(extraCapabilities = list(marionette = TRUE))
remDr$open()
remDr$close()
selServ$stop()
remDr <- remoteDriver(extraCapabilities = list(marionette = TRUE))
source("00-course-setup.r")
remDr <- remoteDriver(extraCapabilities = list(marionette = TRUE))
remDr$open()
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr$navigate(url)
checkForServer() ## deprecated; but gives advice on how to source/start a server
startServer()  ## deprecated
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr$open()
source("00-course-setup.r")
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr$navigate(url)
url <- "http://www.iea.org/policiesandmeasures/renewableenergy/"
remDr$navigate(url)
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > span:nth-child(1)'
regionsElem <- remDr$findElement(using = 'css', value = css)
openRegions <- regionsElem$clickElement() # click on button
url <- "http://www.iea.org/policiesandmeasures/renewableenergy/"
browseURL(url)
remDr$navigate(url)
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > span:nth-child(1)'
regionsElem <- remDr$findElement(using = 'css', value = css)
openRegions <- regionsElem$clickElement() # click on button
remDr$navigate(url)
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > span:nth-child(1)'
regionsElem <- remDr$findElement(using = 'css', value = css)
openRegions <- regionsElem$clickElement() # click on button
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > ul:nth-child(3) > li:nth-child(5) > label:nth-child(1) > input:nth-child(1)'
euElem <- remDr$findElement(using = 'css', value = css)
selectEU <- euElem$clickElement() # click on button
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("2000")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
toDrop <- remDr$findElement(using = 'css', value = css)
clickTo <- toDrop$clickElement() # click on drop-down menu
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
toDrop <- remDr$findElement(using = 'css', value = css)
clickTo <- toDrop$clickElement() # click on drop-down menu
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
toDrop <- remDr$findElement(using = 'css', value = css)
clickTo <- toDrop$clickElement() # click on drop-down menu
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
# click on search button
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
toDrop <- remDr$findElement(using = 'css', value = css)
clickTo <- toDrop$clickElement() # click on drop-down menu
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("1990")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("1990")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("199")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("1")) # enter start year
writeFrom <- fromDrop$sendKeysToElement(list("9")) # enter start year
writeFrom <- fromDrop$sendKeysToElement(list("9")) # enter start year
writeFrom <- fromDrop$sendKeysToElement(list("9")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
remDr$navigate(url)
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > span:nth-child(1)'
regionsElem <- remDr$findElement(using = 'css', value = css)
openRegions <- regionsElem$clickElement() # click on button
# selection "European Union"
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > ul:nth-child(3) > li:nth-child(5) > label:nth-child(1) > input:nth-child(1)'
euElem <- remDr$findElement(using = 'css', value = css)
selectEU <- euElem$clickElement() # click on button
# set time frame
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("2000")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
toDrop <- remDr$findElement(using = 'css', value = css)
clickTo <- toDrop$clickElement() # click on drop-down menu
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
css <- '#main > div:nth-child(1) > form:nth-child(4) > button:nth-child(14)'
searchElem <- remDr$findElement(using = 'css', value = css)
resultsPage <- searchElem$clickElement() # click on button
remDr$closeServer()
content <- read_html("iea-renewables.html", encoding = "utf8")
tabs <- html_table(content, fill = TRUE)
tab <- tabs[[1]]
# add names
names(tab) <- c("title", "country", "year", "status", "type", "target")
head(tab)
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
remDr$navigate(url)
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > span:nth-child(1)'
regionsElem <- remDr$findElement(using = 'css', value = css)
openRegions <- regionsElem$clickElement() # click on button
# selection "European Union"
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > ul:nth-child(3) > li:nth-child(5) > label:nth-child(1) > input:nth-child(1)'
euElem <- remDr$findElement(using = 'css', value = css)
selectEU <- euElem$clickElement() # click on button
# set time frame
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("2000")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > ul:nth-child(3) > li:nth-child(5) > label:nth-child(1) > input:nth-child(1)'
euElem <- remDr$findElement(using = 'css', value = css)
selectEU <- euElem$clickElement() # click on button
selectEU <- euElem$clickElement() # click on button
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("1990")) # enter start year
writeFrom <- fromDrop$sendKeysToElement(list("2000")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
toDrop <- remDr$findElement(using = 'css', value = css)
clickTo <- toDrop$clickElement() # click on drop-down menu
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
css <- '#main > div:nth-child(1) > form:nth-child(4) > button:nth-child(14)'
searchElem <- remDr$findElement(using = 'css', value = css)
resultsPage <- searchElem$clickElement() # click on button
output <- remDr$getPageSource(header = TRUE)
write(output[[1]], file = "iea-renewables.html")
remDr$closeServer()
content <- read_html("iea-renewables.html", encoding = "utf8")
tabs <- html_table(content, fill = TRUE)
tab <- tabs[[1]]
names(tab) <- c("title", "country", "year", "status", "type", "target")
head(tab)
dim(tab)
source("rselenium-2048.r") # by Mark T. Patterson
grand.play()
grand.play()
browseURL("http://www.starbucks.com/store-locator/search/location/chicago")
url <- "http://www.starbucks.com/store-locator/search/location/chicago"
remDr$open()
remDr$navigate(url)
css <- 'span.icon:nth-child(2)'
click_elem <- remDr$findElement(using = 'css', value = css)
css <- 'span.icon:nth-child(2)'
click_elem <- remDr$findElement(using = 'css', value = css)
css <- '.filterButton___TxeLm'
click_elem <- remDr$findElement(using = 'css', value = css)
open_elem <- click_elem$clickElement() # click on button
css <- 'li.item___3yAHj:nth-child(6) > button:nth-child(1)'
click_elem <- remDr$findElement(using = 'css', value = css)
open_elem <- click_elem$clickElement() # click on button
output <- remDr$getPageSource(header = TRUE)
write(output[[1]], file = "starbucks-chicago.html")
remDr$closeServer()
content <- read_html("starbucks-chicago.html", encoding = "utf8")
store_names <- html_nodes(content, ".store-name") %>% html_text()
store_addresses <- html_nodes(content, ".address li:nth-child(1)") %>% html_text()
store_addresses
store_names <- html_nodes(content, ".isActive___38Jkx > div:nth-child(2)") %>% html_text()
store_addresses <- html_nodes(content, ".address li:nth-child(1)") %>% html_text()
store_addresses
store_names <- html_nodes(content, ".truncate") %>% html_text()
store_names
store_addresses <- html_nodes(content, ".addressLine___2afjd , .base___3sH_T div div:nth-child(2) .content___2LtCl .truncate") %>% html_text()
store_addresses
store_addresses <- html_nodes(content, ".addressLine___2afjd:nth-child(1)") %>% html_text()
store_addresses
locations <- paste0(store_addresses, ", Chicago, IL")
locations
pos <- geocode(locations, source = "google")
starbucks_map <- get_map(location=c(lon=mean(pos$lon), lat=mean(pos$lat)), zoom=13, maptype="hybrid")
p <- ggmap(starbucks_map) + geom_point(data=pos, aes(x=lon, y=lat), col="red", size=3)
p
source("00-course-setup.r")
url <- "http://www.iea.org/policiesandmeasures/renewableenergy/"
browseURL(url)
checkForServer() ## deprecated; but gives advice on how to source/start a server
remDr <- remoteDriver(remoteServerAddr = "localhost", port = 4444, browserName = "firefox")
remDr$open()
url <- "http://www.iea.org/policiesandmeasures/renewableenergy/"
remDr$navigate(url)
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > span:nth-child(1)'
regionsElem <- remDr$findElement(using = 'css', value = css)
openRegions <- regionsElem$clickElement() # click on button
openRegions <- regionsElem$clickElement() # click on button
openRegions <- regionsElem$clickElement() # click on button
openRegions <- regionsElem$clickElement() # click on button
openRegions <- regionsElem$clickElement() # click on button
css <- 'div.form-container:nth-child(2) > ul:nth-child(2) > li:nth-child(1) > ul:nth-child(3) > li:nth-child(5) > label:nth-child(1) > input:nth-child(1)'
euElem <- remDr$findElement(using = 'css', value = css)
selectEU <- euElem$clickElement() # click on button
css <- 'div.form-container:nth-child(6) > select:nth-child(2)'
fromDrop <- remDr$findElement(using = 'css', value = css)
clickFrom <- fromDrop$clickElement() # click on drop-down menu
writeFrom <- fromDrop$sendKeysToElement(list("2000")) # enter start year
css <- 'div.form-container:nth-child(6) > select:nth-child(3)'
toDrop <- remDr$findElement(using = 'css', value = css)
clickTo <- toDrop$clickElement() # click on drop-down menu
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
writeTo <- toDrop$sendKeysToElement(list("2010")) # enter end year
css <- '#main > div:nth-child(1) > form:nth-child(4) > button:nth-child(14)'
searchElem <- remDr$findElement(using = 'css', value = css)
resultsPage <- searchElem$clickElement() # click on button
output <- remDr$getPageSource(header = TRUE)
write(output[[1]], file = "iea-renewables.html")
remDr$closeServer()
content <- read_html("iea-renewables.html", encoding = "utf8")
tabs <- html_table(content, fill = TRUE)
tab <- tabs[[1]]
# add names
names(tab) <- c("title", "country", "year", "status", "type", "target")
head(tab)
View(tab)
?remoteDriver
browseURL("http://ropensci.org/")
browseURL("https://github.com/ropensci/opendata")
browseURL("https://cran.r-project.org/web/views/WebTechnologies.html")
browseURL("http://arxiv.org/help/api/index")
browseURL("http://arxiv.org/help/api/user-manual")
browseURL("http://export.arxiv.org/api/query?search_query=all:forecast")
forecast <- read_xml("http://export.arxiv.org/api/query?search_query=all:forecast")
forecast
authors <- xml_find_all(forecast, "//d1:author", ns = xml_ns(forecast))
authors %>% xml_text()
library(aRxiv)
browseURL("http://ropensci.org/tutorials/arxiv_tutorial.html")
ls("package:aRxiv")
?arxiv_search
arxiv_df <- arxiv_search(query = "forecast AND submittedDate:[2016 TO 2017]", limit = 500, output_format = "data.frame")
View(arxiv_df)
arxiv_count('au:"Gary King"')
query_terms
arxiv_count('abs:"political" AND submittedDate:[2016 TO 2017]')
polsci_articles <- arxiv_search('abs:"political" AND submittedDate:[2016 TO 2017]', limit = 500)
library(pageviews)
?article_pageviews
trump_views <- article_pageviews(project = "en.wikipedia", article = "Donald Trump", user_type = "user", start = "2016010100", end = "20161011")
head(trump_views)
clinton_views <- article_pageviews(project = "en.wikipedia", article = "Hillary Clinton", user_type = "user", start = "2016010100", end = "20161011")
plot(ymd_h(trump_views$timestamp), trump_views$views, col = "red", type = "l")
lines(ymd_h(clinton_views$timestamp), clinton_views$views, col = "blue")
trump_views <- article_pageviews(project = "en.wikipedia", article = "Brexit", user_type = "user", start = "2016010100", end = "20161011")
plot(ymd_h(trump_views$timestamp), trump_views$views, col = "red", type = "l")
browseURL("http://www.omdbapi.com/")
title <- "Groundhog Day" %>% URLencode()
title
endpoint <- "http://www.omdbapi.com/?"
omdb_res = GET(url)
res_list <- content(omdb_res, as =  "parsed")
res_list
res_list <- content(omdb_res, as =  "parsed")
res_list
res_list %>% unlist()
res_list %>% unlist() %>% t() %>% data.frame(stringsAsFactors = FALSE)
title <- "Groundhog Day" %>% URLencode()
endpoint <- "http://www.omdbapi.com/?"
url <- paste0(endpoint, "t=", title, "&tomatoes=true")
omdb_res = GET(url)
res_list <- content(omdb_res, as =  "parsed")
res_list %>% unlist() %>% t() %>% data.frame(stringsAsFactors = FALSE)
res_list
res_list %>% unlist()
res_list %>% unlist() %>% t()
res_list %>% unlist() %>% t() %>% data.frame(stringsAsFactors = FALSE)
url <- paste0(endpoint, "s=", title)
omdb_res = GET(url)
res_list <- content(omdb_res, as = "text") %>% jsonlite::fromJSON(flatten = TRUE)
res_list
omdb_res
content(omdb_res)
res_list <- content(omdb_res, as = "text") %>% jsonlite::fromJSON(flatten = TRUE)
res_list
browseURL("https://github.com/hrbrmstr/omdbapi")
browseURL("http://openweathermap.org/current")
browseURL("http://openweathermap.org/api")
apikey <- "&appid=42c7829f663f87eb05d2f12ab11f2b5d"
endpoint <- "http://api.openweathermap.org/data/2.5/find?"
city <- "Chicago,IL"
metric <- "&units=metric"
url <- paste0(endpoint, "q=", city, metric, apikey)
url
weather_res <- GET(url)
res_list <- content(weather_res, as =  "parsed")
res_list
res_list <- content(weather_res, as =  "text")  %>% jsonlite::fromJSON(flatten = TRUE)
res_list
res_list$list
city <- "Mannheim"
url <- paste0(endpoint, "q=", city, metric, apikey)
weather_res <- GET(url)
res_list <- content(weather_res, as =  "text")  %>% jsonlite::fromJSON(flatten = TRUE)
res_list$list
city <- "Mannheim, Germany"
url <- paste0(endpoint, "q=", city, metric, apikey)
weather_res <- GET(url)
res_list <- content(weather_res, as =  "parsed")
res_list <- content(weather_res, as =  "text")  %>% jsonlite::fromJSON(flatten = TRUE)
city <- "Mannheim,Germany"
metric <- "&units=metric"
url <- paste0(endpoint, "q=", city, metric, apikey)
weather_res <- GET(url)
res_list <- content(weather_res, as =  "text")  %>% jsonlite::fromJSON(flatten = TRUE)
res_list$list
api_key <- Sys.getenv("twitter_api_key")
api_secret <- Sys.getenv("twitter_api_secret")
api_key
setup_twitter_oauth(api_key,api_secret) # selecting "1: Yes" usually works
tweets <- searchTwitter(searchString = "Trump", n=25)
tweets_df <- twListToDF(tweets)
head(tweets_df)
tweets
View(tweets_df)
user <- getUser("RDataCollection")
user$name
user$lastStatus
user$followersCount
user$statusesCount
user_followers <- user$getFollowers()
user_followers
user_friends <- user$getFriends()
user_friends
user_timeline <- userTimeline(user, n=20)
user_timeline
timeline_df <- twListToDF(user_timeline)
head(timeline_df)
getCurRateLimitInfo()
?getCurRateLimitInfo
load("../rscraping-intro-duke-2/twitter_auth.RData")
filterStream("tweets_stream.json", track = c("Trump"), timeout = 10, oauth = twitCred)
tweets <- parseTweets("tweets_stream.json", simplify = TRUE)
names(tweets)
cat(tweets$text[1])
View(tweets)