How to apply conditional duplicate removal for data.frame list?

Question

I have a list of data.frame that should have applied a very specific method of removing duplicates. I have a reason for using special conditional duplicate removal for this data.frame list. However, the duplication deletion condition for each individual data frame is different. I want to do a complete duplicate removal for the first list item; for the second element of the list, I need to look for a line that appears more than two times (freq> 2) and contain only one line; for an element of the third list, search for a line that appears more than three times (freq> 3), and save two lines in this data.frame. I am trying to get more software, dynamic solutions for this data processing task. I tried to take a picture to get a good solution, but could not get the desired result.How can i make it easy? Any way to accomplish this task more efficiently for my specific result? Any idea please?

reproducible data.frame:

myList <- list(
    bar= data.frame(start.pos=c(9,19,34,54,70,82,136,9,34,70,136,9,82,136),
                    end.pos=c(14,21,39,61,73,87,153,14,39,73,153,14,87,153),
                    pos.score=c(48,6,9,8,4,15,38,48,9,4,38,48,15,38)),
    cat = data.frame(start.pos=c(7,21,21,72,142,7,16,21,45,72,100,114,142,16,72,114),
                     end.pos=c(10,34,34,78,147,10,17,34,51,78,103,124,147,17,78,124),
                     pos.score=c(53,14,14,20,4,53,20,14,11,20,7,32,4,20,20,32)),
    foo= data.frame(start.pos=c(12,12,12,58,58,58,118,12,12,44,58,102,118,12,58,118),
                    end.pos=c(36,36,36,92,92,92,139,36,36,49,92,109,139,36,92,139),
                    pos.score=c(48,48,48,12,12,12,5,48,48,12,12,11,5,48,12,5))
)

Since it myListis the result of a user-defined function, data.frame cannot be separated. I am looking for a more software solution to do this specific duplicate removal for my data. How can I do a specific removal of duplicates if input is a data.frame list?

my desired result is as follows:

expectedList <- list(
    bar= data.frame(start.pos=c(9,19,34,54,70,82,136),
                    end.pos=c(14,21,39,61,73,87,153),
                    pos.score=c(48,6,9,8,4,15,38)),
    cat= data.frame(start.pos=c(7,21,72,142,7,16,45,100,114,142,16,114),
                    end.pos=c(10,34,78,147,10,17,51,103,124,147,17,124),
                    pos.score=c(53,14,20,4,53,20,11,7,32,4,20,32)),
    foo= data.frame(start.pos=c(12,12,44,58,58,118,102,118,118),
                    end.pos=c(36,36,49,92,92,139,109,139,139),
                    pos.score=c(48,48,12,12,12,5,11,5,5))
)

Edit :

in the second data.frame cat, I'm going to look for lines that appear three times, and save these lines only once; if the line appears twice, I do not duplicate it.

data.frame foo, , , . , data.frame. ?

data.frame? ? !

Answer 1

Map list , , (1:3). data.frame list data.table (setDT(x)), ( "start.pos", "end.pos", "pos.score" ), (.N), if/else , , OP, .I , ($V1) .

library(data.table)
res <- Map(function(x,y) setDT(x)[x[,  .I[if(.N > y) seq_len(pmax(y-1, 1)) 
        else seq_len(.N)]  , .(start.pos, end.pos, pos.score)]$V1], myList, 1:3)
sapply(res, nrow)
#bar cat foo 
#  7  12   9 

sapply(expectedList, nrow) 
#bar cat foo 
#7  12   9 

Answer 2

,

removeDuplicate = function(df, freq=1) {

    # back up the dataframe and add a row id
    tmp = df;
    tmp$cnt = 1:NROW(df);
    # get each row frequency
    cnt = aggregate(cnt~., tmp, length);

    # merge the original data-frame and the row-frequency data-frame
    tmp = merge(df, cnt, by=names(df));
    tmp = rbind(
                tmp[tmp$cnt<=freq, names(df)], # keep all the rows which frequency is not greater than the max allowed
                cnt[, names(df)] # add all the other rows just once
            );

    return(tmp);

}

, :

expectedList = myList
maxFreq = c(1, 2, 3)
for(i in 1:length(expectedList)) {

    expectedList[[i]] = removeDuplicate(expectedList[[i]], maxFreq[i])

}

, , lapply, ...

Answer 3

# Separate individual dataframes
bar = myList$bar 
cat = myList$cat
foo = myList$foo

# We will need ddply command of plyr package
library(plyr)

#Count how many times the rows have repeated and put the value in the fourth column (V1)
bar = ddply(bar,.(start.pos,end.pos,pos.score),nrow)
cat = ddply(cat,.(start.pos,end.pos,pos.score),nrow)
foo = ddply(foo,.(start.pos,end.pos,pos.score),nrow)

# For each data.frame, change the number of repetions to appropriate number of times
# if the rows have repeated for more than the desired number of times
# i.e 1 for bar, 2 for cat, and 3 for foo
for (i in 1:nrow(bar)){
if (bar$V1[i] > 1){
bar$V1[i] = 1
}}
for (i in 1:nrow(cat)){
if (cat$V1[i] > 2){
cat$V1[i] = 1
}}
for (i in 1:nrow(foo)){
if (foo$V1[i] > 2){
foo$V1[i] = 2
}}

# Repeat each row for the number of times indicated in the fourth column.
# This will be 1 for bar, up to 2 for cat, and up to 3 for foo
bar = bar[rep(row.names(bar), bar[,4]), 1:3]
cat = cat[rep(row.names(cat), cat[,4]), 1:3]
foo = foo[rep(row.names(foo), foo[,4]), 1:3]

# Set the rownames to NULL if desired
rownames(cat) = NULL
rownames(bar) = NULL
rownames(foo) = NULL

# Combine the indivudal data.frames into a new list
expectedList = list(bar = bar,cat = cat,foo = foo)