Creating a vector from data.table without using

Question

Let's say I want to create a column in the data.table in which the value in each row is equal to the standard deviations of the values ​​in three other cells on the same row. For example, if I do

DT <- data.table(a = 1:4, b = c(5, 7, 9, 11), c = c(13, 16, 19, 22), d = c(25, 29, 33, 37))
DT
   a  b  c  d
1: 1  5 13 25
2: 2  7 16 29
3: 3  9 19 33
4: 4 11 22 37

and I would like to add a column that contains the standard deviation a, b and d for each row, for example:

   a  b  c  d abdSD
1: 1  5 13 23 12.86
2: 2  7 16 27 14.36
3: 3  9 19 31 15.87
4: 4 11 22 35 17.39

I could, of course, write a for-loop or use the apply function to compute this. Unfortunately, what I really want to do needs to be applied to millions of lines, it is not as simple as calculating the standard deviation, and it needs to be completed in a split second, so I really need a vector solution. I want to write something like

DT[, abdSD := sd(c(a, b, d))]

, , . - data.table, , ? . @Arun

Answer 1

, :

complexFunc <- function(x) sd(x)

cols <- c("a", "b", "d")
rowres <- melt(DT[, rn:=.I], id.vars="rn", variable.factor=FALSE)[, 
    list(abdRes=complexFunc(value[variable %chin% cols])), by=.(rn)]
DT[rowres, on=.(rn)]

3 , -

DT[, abdSD := mapply(complexFunc, a, b, d)]

Answer 2

@Frank, , by=1:nrow(DT)

DT[, abdSD:=sd(c(a,b,d)),by=1:nrow(DT)]

:

   a  b  c  d    abdSD
1: 1  5 13 25 12.85820
2: 2  7 16 29 14.36431
3: 3  9 19 33 15.87451
4: 4 11 22 37 17.38774

---

row_name,

DT$row_id<-row.names(DT)

= row_id ,

DT[, abdSD:=sd(c(a,b,d)),by=row_id]

:

   a  b  c  d row_id    abdSD
1: 1  5 13 25      1 12.85820
2: 2  7 16 29      2 14.36431
3: 3  9 19 33      3 15.87451
4: 4 11 22 37      4 17.38774

row_id, [,row_id:=NULL]

DT[, abdSD:=sd(c(a,b,d)),by=row_id][,row_id:=NULL]

,

   a  b  c  d    abdSD
1: 1  5 13 25 12.85820
2: 2  7 16 29 14.36431
3: 3  9 19 33 15.87451
4: 4 11 22 37 17.38774

.

data.frame , data.table , .

,

Answer 3

, matrixStats package

library(matrixStats)

#sample data
dt <- data.table(a = 1:4, b = c(5, 7, 9, 11), c = c(13, 16, 19, 22), d = c(25, 29, 33, 37))

dt[, `:=`(abdSD = rowSds(as.matrix(.SD), na.rm=T)), .SDcols=c('a','b','d')]
dt

:

   a  b  c  d    abdSD
1: 1  5 13 25 12.85820
2: 2  7 16 29 14.36431
3: 3  9 19 33 15.87451
4: 4 11 22 37 17.38774

Answer 4

, apply , Prem :

40 000 , :

library(matrixStats)

#sample data
dt <- data.table(a = 1:40000, b = rep(c(5, 7, 9, 11),10000), c = rep(c(13, 16, 19, 22),10000), d = rep(c(25, 29, 33, 37),10000))
df <- data.frame(a = 1:40000, b = rep(c(5, 7, 9, 11),10000), c = rep(c(13, 16, 19, 22),10000), d = rep(c(25, 29, 33, 37),10000))

t0 = Sys.time()
dt[, `:=`(abdSD = rowSds(as.matrix(.SD), na.rm=T)), .SDcols=c('a','b','d')]
print(paste("Time taken for data table operation = ",Sys.time() - t0))
# [1] "Time taken for data table operation =  0.117115020751953"


t0 = Sys.time()
df$abdSD <- apply(df[,c("a","b","d")],1, function(x){sd(x)})
print(paste("Time taken for apply opertaion = ",Sys.time() - t0))
# [1] "Time taken for apply opertaion =  2.93488311767578"

DT matrixStats

Answer 5

sd :

vecSD = function(x) {
  n = ncol(x)
  sqrt((n/(n-1)) * (Reduce(`+`, x*x)/n - (Reduce(`+`, x)/n)^2))
}

DT[, vecSD(.SD), .SDcols = c('a', 'b', 'd')]
#[1] 12.85820 14.36431 15.87451 17.38774