Merge branch 'main' of gitlab.liu.se:mehce338-felra653-sigjo290/tdde01-ht24

lab3/Lab3Block1_2021_SVMs_St.R

-# Lab 3 block 1 of 732A99/TDDE01/732A68 Machine Learning
-# Author: jose.m.pena@liu.se
-# Made for teaching purposes
-
-library(kernlab)
-set.seed(1234567890)
-
-data(spam)
-foo <- sample(nrow(spam))
-spam <- spam[foo,]
-tr <- spam[1:3000, ]
-va <- spam[3001:3800, ]
-trva <- spam[1:3800, ]
-te <- spam[3801:4601, ] 
-
-by <- 0.3
-err_va <- NULL
-for(i in seq(by,5,by)){
-  filter <- ksvm(type~.,data=tr,kernel="rbfdot",kpar=list(sigma=0.05),C=i,scaled=FALSE)
-  mailtype <- predict(filter,va[,-58])
-  t <- table(mailtype,va[,58])
-  err_va <-c(err_va,(t[1,2]+t[2,1])/sum(t))
-}
-
-filter0 <- ksvm(type~.,data=tr,kernel="rbfdot",kpar=list(sigma=0.05),C=which.min(err_va)*by,scaled=FALSE)
-mailtype <- predict(filter0,va[,-58])
-t <- table(mailtype,va[,58])
-err0 <- (t[1,2]+t[2,1])/sum(t)
-err0
-
-filter1 <- ksvm(type~.,data=tr,kernel="rbfdot",kpar=list(sigma=0.05),C=which.min(err_va)*by,scaled=FALSE)
-mailtype <- predict(filter1,te[,-58])
-t <- table(mailtype,te[,58])
-err1 <- (t[1,2]+t[2,1])/sum(t)
-err1
-
-filter2 <- ksvm(type~.,data=trva,kernel="rbfdot",kpar=list(sigma=0.05),C=which.min(err_va)*by,scaled=FALSE)
-mailtype <- predict(filter2,te[,-58])
-t <- table(mailtype,te[,58])
-err2 <- (t[1,2]+t[2,1])/sum(t)
-err2
-
-filter3 <- ksvm(type~.,data=spam,kernel="rbfdot",kpar=list(sigma=0.05),C=which.min(err_va)*by,scaled=FALSE)
-mailtype <- predict(filter3,te[,-58])
-t <- table(mailtype,te[,58])
-err3 <- (t[1,2]+t[2,1])/sum(t)
-err3
-
-# Questions
-
-# 1. Which filter do we return to the user ? filter0, filter1, filter2 or filter3? Why?
-
-# 2. What is the estimate of the generalization error of the filter returned to the user? err0, err1, err2 or err3? Why?
-
-# 3. Implementation of SVM predictions.
-
-gaussian_kernel <- function(x_i, x_star, sigma) {
-  return(exp(-sum((x_i - x_star)^2) / (2 * sigma^2)))
-}
-
-sv<-alphaindex(filter3)[[1]]
-co<-coef(filter3)[[1]]
-inte<- - b(filter3)
-k<-NULL
-for(i in 1:10){ # We produce predictions for just the first 10 points in the dataset.
-  k2<-0
-  test_point <- spam[i, -58]
-  for(j in 1:length(sv)){
-    
-    support_vector <- spam[sv[j], -58]
-    
-    kernel_value <- gaussian_kernel(support_vector, test_point, sigma = 0.05) 
-
-    k2 <- k2 + co[j] * kernel_value
-    
-  }
-  k2 <- k2 + inte
-  k <- c(k, sign(k2))
-  
-}
-k
-predict(filter3,spam[1:10,-58], type = "decision")

lab3/assignment2.R

@@ -13,7 +13,7 @@ h_time <- 4
 
 a <- 58.4274
 b <- 14.826
-date <- "1960-07-11"
+date <- "1966-12-10"
 times <- c(
   "04:00:00",
   "06:00:00",
@@ -29,28 +29,41 @@ times <- c(
 )
 
 
-st <- st[st$date <= date, ] # Filter out posterior dates
 st$time <- strptime(st$time, format = "%H:%M:%S")
 
-dist = seq(0, 300000)
-plot(exp(-abs(dist) ^ 2 / (2 * h_distance ^ 2)),
-     type = 'l',
-     xlab = "Physical distance",
-     ylab = "Kernel")
+gaussian_kernel <- function(x, h) {
+  exp(-(x ^ 2) / (2 * h ^ 2))
+}
+
+x = seq(0, 300000, length.out = 1000)
+plot(
+  gaussian_kernel(x, h_distance),
+  type = 'l',
+  xlab = "Physical distance",
+  ylab = "Kernel value",
+  main = "Gaussian distance kernel"
+)
+grid()
 
-#choosing appropriate smoothing coefficient for days
-dist = seq(0, 100)
-plot(exp(-abs(dist) ^ 2 / (2 * h_date ^ 2)),
-     type = 'l',
-     xlab = "Distance in days",
-     ylab = "Kernel")
+x = seq(0, 60)
+plot(
+  gaussian_kernel(x, h_date),
+  type = 'l',
+  xlab = "Distance in days",
+  ylab = "Kernel value",
+  main = "Gaussian date kernel"
+)
+grid()
 
-#choosing appropriate smoothing coefficient for hours
-dist = seq(0, 18)
-plot(exp(-abs(dist) ^ 2 / (2 * h_time ^ 2)),
-     type = 'l',
-     xlab = "Distance in hours",
-     ylab = "Kernel")
+x = seq(0, 18)
+plot(
+  gaussian_kernel(x, h_time),
+  type = 'l',
+  xlab = "Distance in hours",
+  ylab = "Kernel value",
+  main = "Gaussian time kernel"
+)
+grid()
 
 temp_add <- c()
 temp_mult <- c()
@@ -59,6 +72,7 @@ for (time in times) {
   time <- strptime(time, format = "%H:%M:%S")
   
   
+  # Filter out posterior dates and time
   st_temp <- st[st$date < date |
                   (st$date == date &
                      st$time <= time), ]
@@ -66,26 +80,20 @@ for (time in times) {
   
   distance_kernels <- mapply(function(lat, lon) {
     dist <- distHaversine(c(a, b), c(lat, lon))
-    exp(-abs(dist) ^ 2 / (2 * h_distance ^ 2))
-  }, st_temp$latitude, st$longitude)
-  
+    gaussian_kernel(dist, h_distance)
+  }, st_temp$latitude, st_temp$longitude)
   
   
-  date_kernels <- st_temp$date
-  date_kernels <- sapply(date_kernels, function(x_i) {
+  date_kernels <- mapply(function(x_i) {
     dist <- as.numeric(as.Date(date) - as.Date(x_i))
-    exp(-abs(dist) ^ 2 / (2 * h_date ^ 2))
-  })
+    gaussian_kernel(dist, h_date)
+  }, st_temp$date)
   
   
-  time_kernels <- st_temp$time
-  time_kernels <- sapply(time_kernels, function(x_i) {
+  time_kernels <- mapply(function(x_i) {
     dist <- as.numeric(difftime(time, x_i, units = "hours"))
-    exp(-abs(dist) ^ 2 / (2 * h_time ^ 2))
-  })
-  
-  
-  
+    gaussian_kernel(dist, h_time)
+  }, st_temp$time)
   
   
   kernels_add <- distance_kernels + date_kernels + time_kernels
@@ -94,10 +102,36 @@ for (time in times) {
   temp_add <<- c(temp_add,
                  sum(kernels_add * st_temp$air_temperature) / sum(kernels_add))
   temp_mult <<- c(temp_mult,
-                   sum(kernels_mult * st_temp$air_temperature) / sum(kernels_mult))
-  
+                  sum(kernels_mult * st_temp$air_temperature) / sum(kernels_mult))
   
 }
 
-plot(temp_add, type = "o")
-plot(temp_mult, type = "o")
\ No newline at end of file
+plot(
+  temp_add,
+  type = "o",
+  xaxt = "n",
+  xlab = "Time",
+  ylab = "Temperature",
+  main = "Added kernels"
+)
+axis(
+  1,                              
+  at = 1:length(times),         
+  labels = times                  
+)
+grid()
+
+plot(
+  temp_mult,
+  type = "o",
+  xaxt = "n",
+  xlab = "Time",
+  ylab = "Temperature",
+  main = "Multiplied kernels"
+)
+axis(
+  1,                              
+  at = 1:length(times),         
+  labels = times                  
+)
+grid()
\ No newline at end of file

lab3/assignment3.R

+# Lab 3 block 1 of 732A99/TDDE01/732A68 Machine Learning
+# Author: jose.m.pena@liu.se
+# Made for teaching purposes
+
+library(kernlab)
+set.seed(1234567890)
+
+data(spam)
+foo <- sample(nrow(spam))
+spam <- spam[foo, ]
+tr <- spam[1:3000, ]
+va <- spam[3001:3800, ]
+trva <- spam[1:3800, ]
+te <- spam[3801:4601, ]
+
+by <- 0.3
+err_va <- NULL
+for (i in seq(by, 5, by)) {
+  filter <- ksvm(
+    type ~ .,
+    data = tr,
+    kernel = "rbfdot",
+    kpar = list(sigma = 0.05),
+    C = i,
+    scaled = FALSE
+  )
+  mailtype <- predict(filter, va[, -58])
+  t <- table(mailtype, va[, 58])
+  err_va <- c(err_va, (t[1, 2] + t[2, 1]) / sum(t))
+}
+
+filter0 <- ksvm(
+  type ~ .,
+  data = tr,
+  kernel = "rbfdot",
+  kpar = list(sigma = 0.05),
+  C = which.min(err_va) * by,
+  scaled = FALSE
+)
+mailtype <- predict(filter0, va[, -58])
+t <- table(mailtype, va[, 58])
+err0 <- (t[1, 2] + t[2, 1]) / sum(t)
+err0
+
+filter1 <- ksvm(
+  type ~ .,
+  data = tr,
+  kernel = "rbfdot",
+  kpar = list(sigma = 0.05),
+  C = which.min(err_va) * by,
+  scaled = FALSE
+)
+mailtype <- predict(filter1, te[, -58])
+t <- table(mailtype, te[, 58])
+err1 <- (t[1, 2] + t[2, 1]) / sum(t)
+err1
+
+filter2 <- ksvm(
+  type ~ .,
+  data = trva,
+  kernel = "rbfdot",
+  kpar = list(sigma = 0.05),
+  C = which.min(err_va) * by,
+  scaled = FALSE
+)
+mailtype <- predict(filter2, te[, -58])
+t <- table(mailtype, te[, 58])
+err2 <- (t[1, 2] + t[2, 1]) / sum(t)
+err2
+
+filter3 <- ksvm(
+  type ~ .,
+  data = spam,
+  kernel = "rbfdot",
+  kpar = list(sigma = 0.05),
+  C = which.min(err_va) * by,
+  scaled = FALSE
+)
+mailtype <- predict(filter3, te[, -58])
+t <- table(mailtype, te[, 58])
+err3 <- (t[1, 2] + t[2, 1]) / sum(t)
+err3
+
+# Questions
+
+# 1. Which filter do we return to the user ? filter0, filter1, filter2 or filter3? Why?
+
+# 2. What is the estimate of the generalization error of the filter returned to the user? err0, err1, err2 or err3? Why?
+
+# 3. Implementation of SVM predictions.
+
+gaussian_kernel <- function(x_i, x_star, sigma) {
+  exp(-(dist(rbind(x_i, x_star)) ^ 2) / (2 * sigma ^ 2))
+}
+
+sv <- alphaindex(filter3)[[1]]
+co <- coef(filter3)[[1]]
+inte <- -b(filter3)
+k <- NULL
+for (i in 1:10) {
+  # We produce predictions for just the first 10 points in the dataset.
+  
+  k2 <- 0
+  data_point <- spam[i, -58]
+  
+  for (j in 1:length(sv)) {
+    support_vector <- spam[sv[j], -58]
+    kernel_value <- gaussian_kernel(support_vector, data_point, sigma = 0.05)
+    k2 <- k2 + co[j] * kernel_value
+    
+  }
+  
+  k2 <- k2 + inte
+  print(k2)
+  k <- c(k, sign(k2))
+  
+}
+
+# Only first correct, close to decision boundary (0.006292512).
+k
+predict(filter3, spam[1:10, -58], type = "decision")
\ No newline at end of file

lab3/assignment4.R

+library(neuralnet)
+set.seed(1234567890)
+
+
+Var <- runif(500, 0, 10)
+
+
+mydata <- data.frame(Var, Sin=sin(Var))
+
+
+tr <- mydata[1:25,] # Training
+te <- mydata[26:500,] # Test
+
+
+# Random initialization of the weights in the interval [-1, 1]
+winit <- runif(10,-1,1)
+formula <- Sin ~ Var
+  nn <- neuralnet( formula , data = tr, hidden = c(10), startweights = winit )
+    # Plot of the training data (black), test data (blue), and predictions (red)
+    plot(tr, cex=2)
+    points(te, col = "blue", cex=1)
+    points(te[,1],predict(nn,te), col="red", cex=1)
+
+### PART 2 ###
+    
+    
+h1 <- function(x) {
+  x
+}
+    
+    
+h2 <- function(x) {
+  ifelse(x>0,x,0)
+}
+
+h3 <- function(x)  {
+  log(1 + exp(x))
+}
+
+
+
+nn_h1 <- neuralnet( formula , data = tr, hidden = c(10), startweights = t(winit), act.fct = h1 )
+# Plot of the training data (black), test data (blue), and predictions (red)
+plot(tr, cex=2, main = "h1")
+points(te, col = "blue", cex=1)
+points(te[,1],predict(nn_h1,te), col="red", cex=1)
+
+
+
+nn_h2 <- neuralnet( formula , data = tr, hidden = c(10), startweights = t(winit), act.fct = h2 )
+# Plot of the training data (black), test data (blue), and predictions (red)
+plot(tr, cex=2, main="h2")
+points(te, col = "blue", cex=1)
+points(te[,1],predict(nn_h2,te), col="red", cex=1)
+
+
+
+
+nn_h3 <- neuralnet( formula , data = tr, hidden = c(10), startweights = t(winit), act.fct = h3 )
+# Plot of the training data (black), test data (blue), and predictions (red)
+plot(tr, cex=2, main = "h3")
+points(te, col = "blue", cex=1)
+points(te[,1],predict(nn_h3,te), col="red", cex=1)
+
+
+# part 3
+
+
+Var1 <- runif(500, 0, 50)
+
+
+mydata1 <- data.frame(Var = Var1, Sin=sin(Var1))
+
+plot(mydata1, cex=2, main = "500 random points",ylim = c(-10,10))
+points(mydata1, col = "blue", cex=1)
+pred <- predict(nn,te)
+prediciton <- predict(nn,mydata1)
+points(mydata1[,1],prediciton, col="red", cex = 1)
+
+
+# Part 4
+
+nn$weights
+
+
+# Part 5 
+
+
+
+Var2 <- runif(500, 0, 10)
+
+mydata2 <- data.frame(Sin2=sin(Var2), Var2)
+
+formula1 <- Sin2 ~ Var2
+
+nn2 <- neuralnet( formula1 , data = mydata2, hidden = c(10), startweights = winit)
+
+
+
+
+
+plot(mydata2, cex=2, main = "500 random points reverse", ylim = c(-2,10))
+points(mydata2, col = "blue", cex=1)
+points(mydata2[,1],predict(nn2,mydata2), col="red", cex = 1)