作者celestialgod (攸藍)
看板R_Language
標題Re: [問題] 想利用data.table將Rawdata切割欄位
時間Mon Mar 2 10:44:06 2015
改成用CharacterMatrix做為output
省去使用rbind的時間
PS: 之前對CharacterMatrix不甚熟悉，才沒有使用。
library(data.table)
library(magrittr)
library(Rcpp)
library(inline)
sourceCpp(code = '
#include <Rcpp.h>
using namespace Rcpp;
// [[Rcpp::export]]
CharacterMatrix dat_split_f( std::vector< std::string > strings,
NumericVector loc) {
int loc_len = loc.size(), num_strings = strings.size();
CharacterMatrix output(num_strings, loc_len);
for( int j=0; j < num_strings; j++ )
{
for (int i=0; i < loc_len-1; i++)
output(j, i) = strings[j].substr(loc[i], loc[i+1] - loc[i]);
}
return output;
}')
# 測試一下40萬的case 如果要用下面資料記得他們是一行...
dat = fread(paste0(rep("001female2019920404\n002male 3019920505\n003male
4019920606\n004female5019920707\n", 100000), collapse=""),sep="\n", sep2="",
header=FALSE)
tt = proc.time()
dat_split = dat_split_f(dat[[1]], c(0, 3, 9, 11, 19)) %>% data.table()
dat_split[, ':='(V1 = as.numeric(V1), V3 = as.numeric(V3))]
proc.time() - tt
# user system elapsed
# 0.97 0.06 1.03
# 測試一下400萬的case 如果要用下面資料記得他們是一行...
dat = fread(paste0(rep("001female2019920404\n002male 3019920505\n003male
4019920606\n004female5019920707\n", 1000000), collapse=""),sep="\n", sep2="",
header=FALSE)
tt = proc.time()
dat_split = dat_split_f(dat[[1]], c(0, 3, 9, 11, 19)) %>% data.table()
dat_split[, ':='(V1 = as.numeric(V1), V3 = as.numeric(V3))]
proc.time() - tt
# user system elapsed
# 7.73 0.21 7.98
# 十倍的資料量，時間只花八倍，這樣的結果好多了。
windows 64bit R-3.1.2
[email protected]
用了一下
[email protected]的CPU跑40萬的case
# user system elapsed
# 1.14 0.05 1.19
補上用regular expression的方法
library(data.table)
library(plyr)
library(dplyr)
library(magrittr)
dat = fread(paste0(rep("001female2019920404\n002male 3019920505\n003male
4019920606\n004female5019920707\n", 100000), collapse=""),sep="\n",
sep2="",header=FALSE)
tt = proc.time()
dat_regex = dat %>% select(V1) %>% extract2(1) %>%
regexec("([0-9]{3})(female|male\\s{2})([0-9]{2})([0-9]{8})", text = .)
dat_split = dat %>% select(V1) %>% extract2(1) %>%
regmatches(dat_regex) %>% do.call(rbind, .) %>% data.table() %>%
select(2:ncol(.)) %>% setnames(c("id", "gender", "age", "birthday"))
proc.time() - tt
# user system elapsed
# 14.99 0.34 23.76
with
[email protected]
--
※ 發信站: 批踢踢實業坊(ptt.cc), 來自: 36.235.152.127
※ 文章網址: https://webptt.com/m.aspx?n=bbs/R_Language/M.1425264248.A.3E5.html
1F：推 sacidoO: 有試有成功 感謝C大分享 用您的CODE真的超快速的 推~~ 03/03 02:19
※ 編輯: celestialgod (61.224.150.158), 03/07/2015 21:36:34
※ 編輯: celestialgod (61.224.150.158), 03/07/2015 21:39:05