data_cleaning.Rmd

---
title: "BDA Project: Malicious and Benign Website URL detections"
author: "Nguyen Xuan Binh, Duong Le"
date: "January 2023"
output: 
  pdf_document:
    toc: yes
    toc_depth: 3
    fig_caption: yes
bibliography: bibliography.bib
---

```{r include=FALSE}
library(rstan)
library(cmdstanr)
library(ggplot2)
library(dplyr)
library(tidyr)
library(grid)
library(gridExtra)
library(scales)
library(loo)
library(sentimentr)
library(stringr)
library(gridExtra)
library(MASS)
options(dplyr.summarise.inform = FALSE)
```


```{r}
# Load the dataframe
train_websites <- read.csv("websites/Webpages_Classification_train_data.csv")
test_websites <- read.csv("websites/Webpages_Classification_test_data.csv")
train_websites <- na.omit(train_websites)
test_websites <- na.omit(test_websites)
```

```{r}
# Filter the test_websites to keep only rows where label is "bad"
bad_rows <- train_websites %>% filter(label == "bad")

# Randomly select 300 rows from bad rows
bad_sample_rows <- bad_rows %>% sample_n(30)

# Filter the test_websites to keep only rows where label is "good"
good_rows <- train_websites %>% filter(label == "good")

# Randomly select 1200 rows from good_rows
good_sample_rows <- good_rows %>% sample_n(90)

# Concatenate bad_rows and sample_rows to create the new train_websites
train_websites <- rbind(bad_sample_rows, good_sample_rows)
```

```{r}
# Filter the test_websites to keep only rows where label is "bad"
bad_rows <- test_websites %>% filter(label == "bad")

# Randomly select 300 rows from bad rows
bad_sample_rows <- bad_rows %>% sample_n(50)

# Filter the test_websites to keep only rows where label is "good"
good_rows <- test_websites %>% filter(label == "good")

# Randomly select 1200 rows from good_rows
good_sample_rows <- good_rows %>% sample_n(200)

# Concatenate bad_rows and sample_rows to create the new test_websites
test_websites <- rbind(bad_sample_rows, good_sample_rows)
```

```{r}
print(sum(train_websites$label=="bad"))
print(sum(train_websites$label=="good"))

print(sum(test_websites$label=="bad"))
print(sum(test_websites$label=="good"))
```
```{r}
# Define the special characters you want to count
special_chars <- c("/","%", "#", "&”, “." , "," ,"=")

# Create a new column "num_special" and populate it with the number of special characters in each URL

train_websites$num_special <- sapply(train_websites$url, function(x) sum(str_count(x, paste(special_chars, collapse="|"))))

test_websites$num_special <- sapply(test_websites$url, function(x) sum(str_count(x, paste(special_chars, collapse="|"))))
```

```{r}
print(colnames(train_websites))

keptColNames = c("label", "url_len", "geo_loc", "https", "js_len", "js_obf_len", "who_is", "num_special")
train_websites <- train_websites[, keptColNames]

keptColNames = c("label", "url_len", "geo_loc", "https", "js_len", "js_obf_len", "who_is", "num_special")
test_websites <- test_websites[, keptColNames]

print(colnames(train_websites))
```

```{r}
print(head(train_websites))
```

```{r}
write.csv(train_websites, "websites/train_websites.csv")
write.csv(test_websites, "websites/test_websites.csv")
```

```{r}
train_websites <- read.csv("websites/train_websites.csv")
test_websites <- read.csv("websites/test_websites.csv")
```

```{r}
# Count the number of rows for each combination of https and who_is

train_websites_count <- train_websites %>% 
  filter(label == "good") 
numberOfMaliciousURLs <- nrow(train_websites_count)
  
train_websites_count <- train_websites %>% 
  filter(label == "good") %>% 
  group_by(https) %>%
  summarize(count = n())

# Create a pie chart with a legend
p1 <- ggplot(train_websites_count, aes(x = "", y = count, fill = interaction(https))) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0, direction = -1) +
  scale_fill_manual(values = c("red", "green"), labels=c("no","yes")) +
  theme(legend.position = "bottom") +
  theme(plot.title = element_text(hjust = 0.5)) +
  ggtitle(paste("HTTPS in",numberOfMaliciousURLs,"benign URLs\n(yes/no for HTTPS)")) +
  
  guides(fill=guide_legend(title=""))

# Count the number of rows for each combination of https and who_is

train_websites_count <- train_websites %>% 
  filter(label == "good") 
numberOfBenignURLs <- nrow(train_websites_count)
  
train_websites_count <- train_websites %>% 
  filter(label == "good") %>% 
  group_by(who_is) %>%
  summarize(count = n()) %>% 
  arrange(desc(count))

# Create a pie chart with a legend
p2 <- ggplot(train_websites_count, aes(x = "", y = count, fill = interaction(who_is))) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0, direction = 1) +
  scale_fill_manual(values = c("green", "red"), labels=c("complete","incomplete")) +
  theme(legend.position = "bottom") +
  theme(plot.title = element_text(hjust = 0.5)) +
  ggtitle(paste("WHOIS in",numberOfBenignURLs,"benign URLs\n(complete/incomplete for WHOIS)")) +
  
  guides(fill=guide_legend(title=""))


grid.arrange(p1, p2, ncol = 2)
```

```{r}
# Count the number of rows for each combination of https and who_is

train_websites_count <- train_websites %>% 
  filter(label == "good") 
numberOfBenignURLs <- nrow(train_websites_count)
  
train_websites_count <- train_websites %>% 
  filter(label == "good") %>% 
  group_by(https, who_is) %>%
  summarize(count = n())

# Create a pie chart with a legend
ggplot(train_websites_count, aes(x = "", y = count, fill = interaction(https, who_is))) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0, direction = 1) +
  scale_fill_manual(values = c("blue", "green", "red", "orange")) +
  theme(legend.position = "bottom") +
  theme(plot.title = element_text(hjust = 0.5)) +
  ggtitle(paste("(HTTPS.WHOIS) pair combination in",numberOfBenignURLs,"benign URLs\n(yes/no for HTTPS and complete/incomplete for WHOIS)")) +
  
  guides(fill=guide_legend(title=""))
```

```{r}
# Count the number of rows for each combination of https and who_is

train_websites_count <- train_websites %>% 
  filter(label == "bad") 
numberOfMaliciousURLs <- nrow(train_websites_count)
  
train_websites_count <- train_websites %>% 
  filter(label == "bad") %>% 
  group_by(https) %>%
  summarize(count = n())

# Create a pie chart with a legend
p1 <- ggplot(train_websites_count, aes(x = "", y = count, fill = interaction(https))) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0, direction = 1) +
  scale_fill_manual(values = c("red", "green"), labels=c("no","yes")) +
  theme(legend.position = "bottom") +
  theme(plot.title = element_text(hjust = 0.5)) +
  ggtitle(paste("HTTPS in",numberOfMaliciousURLs,"malicious URLs\n(yes/no for HTTPS)")) +
  
  guides(fill=guide_legend(title=""))

# Count the number of rows for each combination of https and who_is

train_websites_count <- train_websites %>% 
  filter(label == "bad") 
numberOfMaliciousURLs <- nrow(train_websites_count)
  
train_websites_count <- train_websites %>% 
  filter(label == "bad") %>% 
  group_by(who_is) %>%
  summarize(count = n()) %>% 
  arrange(desc(count))

# Create a pie chart with a legend
p2 <- ggplot(train_websites_count, aes(x = "", y = count, fill = interaction(who_is))) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0, direction = -1) +
  scale_fill_manual(values = c("green", "red"), labels=c("complete","incomplete")) +
  theme(legend.position = "bottom") +
  theme(plot.title = element_text(hjust = 0.5)) +
  ggtitle(paste("WHOIS in",numberOfMaliciousURLs,"malicious URLs\n(complete/incomplete for WHOIS)")) +
  
  guides(fill=guide_legend(title=""))


grid.arrange(p1, p2, ncol = 2)
```

```{r}
# Count the number of rows for each combination of https and who_is

train_websites_count <- train_websites %>% 
  filter(label == "bad") 
numberOfMaliciousURLs <- nrow(train_websites_count)
  
train_websites_count <- train_websites %>% 
  filter(label == "bad") %>% 
  group_by(https, who_is) %>%
  summarize(count = n())

# Create a pie chart with a legend
ggplot(train_websites_count, aes(x = "", y = count, fill = interaction(https, who_is))) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0, direction = 1) +
  scale_fill_manual(values = c("blue", "green", "red", "orange")) +
  theme(legend.position = "bottom") +
  theme(plot.title = element_text(hjust = 0.5)) +
  ggtitle(paste("(HTTPS.WHOIS) pair combination in",numberOfMaliciousURLs,"malicious URLs\n(yes/no for HTTPS and complete/incomplete for WHOIS)")) +
  
  guides(fill=guide_legend(title=""))
```

```{r}
ggplot(data = train_websites %>% filter(label == "good") , aes(x = js_len, y = js_obf_len)) +
  geom_point() +
  ggtitle("num_special vs url_len") +
  xlab("js_len") +
  ylab("js_obf_len") +
  theme(plot.title = element_text(hjust = 0.5))
```

```{r}
ggplot(data = train_websites, aes(x = js_len, y = js_obf_len, color = label)) +
  geom_point() +
  scale_color_manual(values = c("red", "blue"), 
                     labels = c("malicious", "benign"),
                     guide = guide_legend(title = "Label")) +
  ggtitle("js_len vs js_obf_len") +
  xlab("js_len") +
  ylab("js_obf_len") +
  theme(plot.title = element_text(hjust = 0.5)) +
  geom_vline(xintercept = 250, linetype = "dashed", color = "black") + 
  geom_hline(yintercept = 100, linetype = "dashed", color = "black") +
  annotate("text", x = 260, y = Inf, label = "js_len = 250", hjust = 0, vjust = 1) +
  annotate("text", x = Inf, y = 60, label = "js_ofs_len = 100", hjust = 1, vjust = 0) 
  #guides(color = guide_legend(title = "Label"))
```




```{r}
hist(train_websites$js_len, main = "JS length histogram of the recorded URLs", breaks = 100)
hist(train_websites$js_obf_len, main ="Obfuscated JS length histogram of the recorded URLs", breaks = 100)
```

```{r}
# Group the dataframe by geo_loc and count the number of rows for each country
train_websites_count <- train_websites %>% 
  group_by(geo_loc) %>%
  summarize(count = n() ) %>%
  top_n(3, count) %>%
  slice_tail(n=3)

# Count the number of benign and malicious URLs for each country
train_websites_count_label <- train_websites %>% 
  filter(geo_loc %in% train_websites_count$geo_loc) %>%
  group_by(geo_loc, label) %>%
  summarize(count = n())

# Plot the bar chart
ggplot(train_websites_count_label, aes(x = geo_loc, y = count, fill = label)) +
  geom_bar(stat = "identity", position = "stack") +
  scale_fill_manual(values = c("red", "blue"), 
                    labels = c("malicious","benign")) +
  xlab("Country") +
  ylab("Number of URLs") +
  ggtitle("Distribution of benign and malicious URLs of top 5 recorded countries") +
  guides(fill = guide_legend(title = "Label")) +
  theme(axis.text.x = element_text(angle = 30, hjust = 0.5, vjust = 0.5, size = 10, 
                                  margin = margin(r = -20, unit = "pt"),
                                  family = "serif", 
                                  lineheight = 0.9, color = "black"))
```

```{r}
train_websites_top_3 <- train_websites %>% 
  filter(geo_loc %in% train_websites_count$geo_loc)

test_websites_top_3 <- test_websites %>% 
  filter(geo_loc %in% train_websites_count$geo_loc)

print(nrow(train_websites_top_3))

print(nrow(test_websites_top_3))
```

```{r}
#train_websites_top_3$safety <- ifelse(train_websites_top_3$https == "yes" & 
#                                train_websites_top_3$who_is == "complete", 1,
#                                ifelse(train_websites_top_3$https == "no" 
#                                & train_websites_top_3$who_is == "complete", 2,
#                                ifelse(train_websites_top_3$https == "yes" & 
#                                train_websites_top_3$who_is == "incomplete", 3, 4)))

#test_websites_top_3$safety <- ifelse(test_websites_top_3$https == "yes" & 
#                                test_websites_top_3$who_is == "complete", 1,
#                                ifelse(test_websites_top_3$https == "no" 
#                                & test_websites_top_3$who_is == "complete", 2,
#                                ifelse(test_websites_top_3$https == "yes" & 
#                                test_websites_top_3$who_is == "incomplete", 3, 4)))

train_websites_top_3 <- rename(train_websites_top_3, whois = who_is)
test_websites_top_3 <- rename(test_websites_top_3, whois = who_is)

train_websites_top_3$https_bin <- ifelse(train_websites_top_3$https == "yes", 0, 1)

train_websites_top_3$whois_bin <- ifelse(train_websites_top_3$whois == "complete", 0, 1)

test_websites_top_3$https_bin <- ifelse(test_websites_top_3$https == "yes", 0, 1)

test_websites_top_3$whois_bin <- ifelse(test_websites_top_3$whois == "complete", 0, 1)


keptColNames = c("label", "geo_loc", "js_len", "js_obf_len", "https", "https_bin", "whois", "whois_bin")
train_websites_top_3 <- train_websites_top_3[, keptColNames]
test_websites_top_3 <- test_websites_top_3[, keptColNames]
print(train_websites_top_3)
print(test_websites_top_3)
```

```{r}
#train_websites_top_3$js <- ifelse(train_websites_top_3$js_len < 250 & train_websites_top_3$js_obf_len >= 100, 0, 
#                                   ifelse(train_websites_top_3$js_len < 250 & #train_websites_top_3$js_obf_len < 100, 1, 2))

#test_websites_top_3$js <- ifelse(test_websites_top_3$js_len < 250 & test_websites_top_3$js_obf_len >= 100, 0, 
#                                   ifelse(test_websites_top_3$js_len < 250 & #test_websites_top_3$js_obf_len < 100, 1, 2))


train_websites_top_3$js_len_bin <- ifelse(train_websites_top_3$js_len < 250, 0, 1)

train_websites_top_3$js_obf_len_bin <- ifelse(train_websites_top_3$js_obf_len < 100, 0, 1)

test_websites_top_3$js_len_bin <- ifelse(test_websites_top_3$js_len < 250, 0, 1)

test_websites_top_3$js_obf_len_bin <- ifelse(test_websites_top_3$js_obf_len < 100, 0, 1)

print(train_websites_top_3)
print(test_websites_top_3)

```

```{r}
print(head(train_websites_top_3))
print(head(test_websites_top_3))
```

```{r}
#train_websites_top_3 <- train_websites_top_3 %>% mutate(label = ifelse(label == "good", 0, 1))
#test_websites_top_3 <- test_websites_top_3 %>% mutate(label = ifelse(label == "good", 0, 1))

train_websites_top_3$label_bin <- ifelse(train_websites_top_3$label == "good", 0, 1)

test_websites_top_3$label_bin <- ifelse(test_websites_top_3$label == "good", 0, 1)

print(head(test_websites_top_3))
```

```{r}
write.csv(train_websites_top_3, "websites/train_websites_top_3.csv")
write.csv(test_websites_top_3, "websites/test_websites_top_3.csv")
```

```{r}
train_websites_top_3 <- read.csv("websites/train_websites_top_3.csv")
test_websites_top_3 <- read.csv("websites/test_websites_top_3.csv")
```