Spot.Rmd

---
title: "Spotify analysis"
author: "Sumeet k Singh"
date: "7/2/2020"
output: github_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

```{r setup2, include=FALSE}
pacman::p_load(dplyr, readr,tidyverse, ggplot2,reshape,corrplot,hexbin,RColorBrewer,caret,plotly,autoplotly,tidyr,forecast,gains,randomForest,caret,rsample,rpart,rpart.plot,devtools,DataExplorer)
```

```{r readfile, echo=FALSE}
spotify <- read.csv("Spotifydata.csv")
head(spotify)

```

This is a dataset consisting of features for tracks fetched using Spotify's Web API. The tracks are labeled '1' or '0' ('Hit' or 'Flop') depending on some criteria of the author.
This dataset can be used to make a classification model that predicts whether a track would be a 'Hit' or not.


			
```{r}
str(spotify)
```

```{r DataCleaning, echo=FALSE, results='hide'}
spotify_data1 <- select(spotify, -1)
spotify_data1
count(spotify_data1,target)
sum(is.na(spotify_data1))
```

```{r}
#if (!require(devtools)) install.packages("devtools")
#library(devtools)
#install_github("boxuancui/DataExplorer")
  
```

```{r EDA}
#DataExplorer::create_report(spotify_data1)


plot_bar(spotify_data1)

plot_str(spotify_data1)

plot_intro(spotify_data1)

plot_histogram(spotify_data1)

plot_correlation(spotify_data1, maxcat = 5L)

```



```{r LinearRigression}

L1_data <- spotify_data1[c(1,2,3,5,6,7,8,9,10,11,12,13,14,15,16)]
corrplot(cor(L1_data[c(1,2,3,5,6,7,8,9,10,11,12,13)]))
set.seed(42)
rows1 <- sample(nrow(L1_data))
d1 <- L1_data[rows1, ]

split <- round(nrow(d1) * 0.8)
train.df1 <- d1[1:split, ]
test.df1 <- d1[(split+1):nrow(d1), ]
round(nrow(train.df1)/nrow(d1), digits = 3)



```
 

```{r LinearRegression1}
spot.lm <- lm(target ~ acousticness + danceability + tempo + time_signature + speechiness + mode + key + liveness + loudness + valence , data = train.df1)

summary(spot.lm)
```


```{r Prediction1}
spot.pred <- predict(spot.lm,test.df1)

pred_cutoff_50<- ifelse(spot.pred > 0.5, 1, 0)

accuracy(spot.pred, test.df1$target)

plot(spot.lm)


conf.matrix <- table(Actual = test.df1$target, Predicted = pred_cutoff_50)
conf.matrix


accuracy_Testlm <- sum(diag(conf.matrix)) / sum(conf.matrix)
print(paste('Accuracy for test', accuracy_Testlm))
```

```{r LogisticRegression1}
logit.reg <- glm(target ~ ., data = train.df1[,c(1:13)], family = "binomial") 

summary(logit.reg)

# Generate odds-ratios
exp(coef(logit.reg))
```

```{r Prediction2}
logit.reg.pred <- predict(logit.reg, test.df1[,c(1:13)], type = "response")


pred_cutoff_lr<- ifelse(logit.reg.pred > 0.5, 1, 0)
accuracy(logit.reg.pred,test.df1$target)

conf.matrix2 <- table(Actual = test.df1$target, Predicted = pred_cutoff_lr)
conf.matrix2


accuracy_Test_lr <- sum(diag(conf.matrix2)) / sum(conf.matrix2)
    accuracy_Test_lr
```

```{r DecisionTree}
dt.fit <- rpart(target~., data = train.df1[,c(1:13)], method = 'class')
rpart.plot(dt.fit, extra = 106)

```

```{r Prediction3}
predict_dt <-predict(dt.fit, test.df1, type = 'class')
table_mat <- table(test.df1$target, predict_dt)
table_mat
accuracy_Testdt <- sum(diag(table_mat)) / sum(table_mat)
print(paste('Accuracy for test', accuracy_Testdt))

```

```{r AccuracyTune fnuction}
accuracy_tune <- function(fit) {
    predict_unseen <- predict(dt.fit, test.df1, type = 'class')
    table_mat <- table(test.df1$target, predict_unseen)
    accuracy_Test <- sum(diag(table_mat)) / sum(table_mat)
    accuracy_Test
}
```

```{r Hyperparametertuning}
control <- rpart.control(minsplit = 3,
    minbucket = round(3 / 3),
    maxdepth = 3,
    cp = 0.01)
tune_fit <- rpart(target~., data = train.df1[,c(1:13)], method = 'class', control = control)
accuracy_tune(tune_fit)
print(paste('Accuracy for test', accuracy_tune(tune_fit)))

```

```{r RandomForest1}
train.df1$target <- as.factor(train.df1$target)
test.df1$target <- as.factor(test.df1$target)
rf.fit1 <- randomForest(target ~ ., data = train.df1[,c(1:13)], importance = TRUE)
rf.fit1

plot(rf.fit1)


```

```{r Prediction3a}
prediction <-predict(rf.fit1, test.df1)
pred<- ifelse(prediction == 1, 1, 0)
conf.matrix_rf <- table(Actual = test.df1$target, Predicted = pred)
conf.matrix_rf

accuracy_Testrf <- sum(diag(conf.matrix_rf)) / sum(conf.matrix_rf)
print(paste('Accuracy for test', accuracy_Testrf))

```
```{r VariableImportancePlot}
varImpPlot(rf.fit1)

varImp(rf.fit1)
```


```{r Loading Dataset with songs from 2000s to 2010s,results='hide'}
dataset_of_10s <- read_csv("dataset-of-10s.csv")
dataset_of_00s <- read_csv("dataset-of-00s.csv")

D1 <- rbind(dataset_of_10s,dataset_of_00s)
str(D1)
```

```{r}
plot_correlation(D1,maxcat = 5L)
summary(D1)
D1 <- subset(D1, select = -c(uri,sections,chorus_hit))

```

```{r DataPartition1, results='hide'}
set.seed(42)
train.rf <- sample(nrow(D1), 0.7*nrow(D1), replace = FALSE)
TrainSet <- D1[train.rf,]
ValidSet <- D1[-train.rf,]
summary(TrainSet)
summary(ValidSet)

TrainSet <- subset(TrainSet, select = -c(track,artist))
ValidSet <- subset(ValidSet, select = -c(track,artist))

```

```{r RandomForest2a}
TrainSet$target <- as.factor(TrainSet$target)
ValidSet$target <- as.factor(ValidSet$target)

model2 <- randomForest(target ~ ., data = TrainSet, importance = TRUE)
model2


plot(model2)

```
```{r Prediction4}
prediction_model2 <-predict(model2, ValidSet)

confusionMatrix(prediction_model2, ValidSet$target)
```

```{r ModelTuning}
trControl <- trainControl(method = "cv",
    number = 10,
search = "grid")
```

```{r Modeltuning1}
rf_default <- train(target~.,
    data = TrainSet,
    method = "rf",
    metric = "Accuracy",
    trControl = trControl)
# Print the results
print(rf_default)
```

```{r Increasing the range of mtry}
set.seed(1234)
tuneGrid <- expand.grid(.mtry = c(2:10))
rf_mtry <- train(target~.,
    data = TrainSet,
    method = "rf",
    metric = "Accuracy",
    tuneGrid = tuneGrid,
    trControl = trControl,
    importance = TRUE,
    nodesize = 14,
    ntree = 300)
print(rf_mtry)
```

```{r SaveBestTreeResult}
best_mtry <- rf_mtry$bestTune$mtry 
best_mtry

```

```{r FindBestnode}
store_maxnode <- list()
tuneGrid <- expand.grid(.mtry = 2)
for (maxnodes in c(5: 20)) {
    set.seed(1234)
    rf_maxnode <- train(target~.,
        data = TrainSet,
        method = "rf",
        metric = "Accuracy",
        tuneGrid = tuneGrid,
        trControl = trControl,
        importance = TRUE,
        nodesize = 14,
        maxnodes = maxnodes,
        ntree = 300)
    current_iteration <- toString(maxnodes)
    store_maxnode[[current_iteration]] <- rf_maxnode
}
results_mtry <- resamples(store_maxnode)
summary(results_mtry)
```

```{r Finding best no of ntree}
store_maxtrees <- list()
for (ntree in c(250, 300, 350, 400, 450, 500, 550, 600, 800, 1000, 2000)) {
    set.seed(5678)
    rf_maxtrees <- train(target~.,
        data = TrainSet,
        method = "rf",
        metric = "Accuracy",
        tuneGrid = tuneGrid,
        trControl = trControl,
        importance = TRUE,
        nodesize = 14,
        maxnodes = 17,
        ntree = ntree)
    key <- toString(ntree)
    store_maxtrees[[key]] <- rf_maxtrees
}
results_tree <- resamples(store_maxtrees)
summary(results_tree)
```

```{r Final fit}
fit_rf <- train(target~.,
    TrainSet,
    method = "rf",
    metric = "Accuracy",
    tuneGrid = tuneGrid,
    trControl = trControl,
    importance = TRUE,
    nodesize = 14,
    ntree = 550,
    maxnodes = 17)
```

```{r Prediction5}
prediction <-predict(fit_rf, ValidSet)

confusionMatrix(prediction, ValidSet$target)

```

```{r VariableImportance1}
plot(varImp(fit_rf))


```

```{r Predict older songs using latest data}
set.seed(42)
Datatrain <- spotify_data1

Datatest <- D1[,c(1,2,9,3,14,4,10,5,11,6,7,8,13,15,12,16)]

Datatest.rf <- sample(nrow(Datatest), 0.5*nrow(Datatest), replace = FALSE)

Datatest.rf1 <- Datatest[Datatest.rf,]



```


```{r PredictOldSongs}

Datatrain$target <- as.factor(Datatrain$target)
older_songs_pred <- randomForest(target ~ ., data = Datatrain[,c(1:14)], importance = TRUE)


old_song_list <-predict(older_songs_pred, Datatest.rf1)
old_pred<- ifelse(old_song_list==1, 1, 0)


conf.matrix.old <- table(Actual = Datatest.rf1$target, Predicted = old_pred )
conf.matrix.old

old_recommend <- cbind(Datatest.rf1[,c(1,2)],old_pred)
list <- old_recommend %>%  filter(old_pred==1)
head(list)


```