Lab3 Web Scraping and API 101

#pip3 install notebook
#jupyter notebook

# get current working directory
import os
path=os.getcwd()
print(path)

# create a new folder for our course
new_path="./soc591/"
os.makedirs(new_path)

# change the current working directory to soc591
os.chdir(new_path,eval=FALSE)
# check the current wd
print("Current wd: ",os.getcwd())

# let us create a new file in current WD, write some texts into the file, and then close it
f = open("soc591.txt",mode="w+")
col_vars = "id;text\n"
f.write(col_vars)
f.write("1;This is a demo for writing some texts\n")
f.close()

# Let us read the soc591.txt file and assign it to variable text_df
text_df = open("soc591.txt", "r").read()
print(text_df)

# list file content
os.listdir(".")

# Let us remove the soc591.txt file
os.remove("soc591.txt")

import pandas as pd
from bs4 import BeautifulSoup as bs
import urllib.parse
import urllib.request

def get_crp_industry_list(url):
  ''' Access Opensecrets.org website and return industry names and ids.
  '''
  # Specify userheader
  userHeader = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12"}
  req = urllib.request.Request(url, headers=userHeader)
  # open url and read web page
  response = urllib.request.urlopen(req)
  the_page = response.read()
  # beautifulsoup parse html
  soup=bs(the_page,"html.parser")
  #print(soup)
  # get all industry links and names
  indList=soup.find("div",{"id":"rightColumn"}).find_all("a")
  #print(indList)
  # clean raw data
  indLinks = []
  indNames = []
  for link in indList[1:]:
      indLinks.append(link['href'].replace("indus.php?ind=",""))
      indNames.append(link.contents[0].strip())
  #print(indLinks,indNames)
  # create a dataset
  indDF = pd.DataFrame({"indLinks":indLinks,"indNames":indNames})
  return indDF

url="https://www.opensecrets.org/industries/slist.php"
data_ind_list=get_crp_industry_list(url)
print(data_ind_list)

#!pip3 install selenium
#pip3 install webdriver_manager
#pip3 install bs4

if (!requireNamespace("pacman"))
  install.packages('pacman')

library(pacman)
packages<-c("tidyverse","tidytext","rvest", "RSelenium","coreNLP",
            "tm","haven","readxl","here","knitr","stopwords")
p_load(packages,character.only = TRUE)

# read tsv file using read_csv
data <- read_tsv(url("https://yongjunzhang.com/files/blm-data.tsv"))

# Show a table for visual check
knitr::kable(data[1:3,],cap="Black Lives Matter Protest (elephrame.com)")

data <- data %>% 
  # split location
  separate(protest_location,c("city","state"),sep = ",",remove = T) %>% 
  # split protest start time
  separate(protest_start,c("day","date","year"),sep = ",",remove = T) %>%
  # clean subjects and participants
  mutate(
    protest_subject = str_replace(protest_subject,"Subject\\(s\\): ",""),
    protest_participants = str_replace(protest_participants,"Participant\\(s\\): ",""),
    protest_time = str_replace(protest_time,"Time: ",""),
    protest_description = str_replace(protest_description,"Description: ",""),
    protest_urls = str_replace(protest_urls,"Source\\(s\\):","")
    )

# Show a table for visual check
knitr::kable(data[1:3,],cap="Black Lives Matter Protest (elephrame.com)")

library(tidytext)

# Let us say we are interested in protest description. We need to restructure it as one-token-per-row format. The unnest_tokens function is a way to convert a dataframe with a text column to be one-token-per-row:

tidy_data <- data %>%
  # protest_urls is messay, let us get rid of it first
  select(-protest_urls) %>% 
  # one token per row. This function uses the tokenizers package to separate each line into words. The default tokenizing is for words, but other options include characters, ngrams, sentences, lines, paragraphs, or separation around a regex pattern.
  unnest_tokens(word, protest_description) %>% 
  # remove stop words
  anti_join(tidytext::get_stopwords("en",source="snowball")) %>% 
  # you can also add your own stop words if you want
  # check here to see tibble data structure <https://tibble.tidyverse.org/>
  anti_join(tibble(word=c("no","details")),by="word")

knitr::kable(tidy_data[1:10,],cap="Black Lives Matter Protest (elephrame.com)")

tidy_data %>% 
  count(word, sort = TRUE) 

library(ggplot2)

tidy_data %>%
  count(word, sort = TRUE) %>%
  filter(n > 100) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(word, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

 data %>%
  select(-protest_urls) %>% 
  unnest_tokens(bigram, protest_description,token = "ngrams", n = 2) %>%
  count(bigram,sort = TRUE)

tidy_words <- tidy_data %>%
  count(protest_id, word, sort = TRUE)

tidy_words

# create a dataset having urls, year, and ids
urls <- data %>% 
  select(protest_id,state,date,protest_urls) %>% 
  # let us extract all protest_urls
  mutate(protest_urls=str_replace_all(protest_urls,"^##|^#","")) %>% 
  separate_rows(protest_urls,sep="##") %>% 
  filter(str_detect(protest_urls,"^http"))

# only keep one url for each protest
urls1 <- urls %>% 
  distinct(protest_id,.keep_all = T)

# let us take a look at the data
knitr::kable(urls1[1:5,],cap="Black Lives Matter Protest (elephrame.com)")

knitr::include_graphics('rvest_httr.png')

library(rvest)
library(httr)

url <- urls1$protest_urls[1]
url

# rvest provides two ways of making request: read_html() and html_session() 
# read_html() can parse a HTML file or an url into xml document. 
# html_session() is built on GET() from httr package.

#making GET request andparse website into xml document
pagesource <- read_html(url)

#using html_session which creates a session and accept httr methods
my_session <- session(url)

#html_session is built upon httr, you can access response with a session
response <- my_session$response

#retrieve content as raw
content_raw <- content(my_session$response,as = "raw")
#retrieve content as text
content_text <- content(my_session$response,as = "text")
#retrieve content as parsed(parsed automatically)
content_parsed <- content(my_session$response,as = "parsed")

# connect to chrome driver
driver <- RSelenium::rsDriver(browser = "chrome",port = 4443L)
remote_driver <- driver[["client"]] 
remote_driver$navigate(url)

# retrieve the article

main_article <- remote_driver$findElement(using = "class", value="p402_premium")

text <- main_article$getElementText()

# let us clean those special characters like \n \t, etc.

tidy_text <- text[[1]] %>% 
  # remove all whitespaces, note it is regex \t
  str_replace_all("\\s"," ") %>% 
  # reove some weird punct
  str_replace_all('\\"',"") %>% 
  # remove some double spaces
  str_squish %>% 
  # reve spaces at the begining and end of the text
  str_trim %>% 
  # lower case
  tolower()

tidy_text