From 982903343362af9b17b78a79c2d4dae9f7b4e6cf Mon Sep 17 00:00:00 2001
From: Schultheis <hschult@kerckhoff.mpg.de>
Date: Tue, 3 Apr 2018 14:15:33 +0200
Subject: [PATCH] parser: don't read whole file during header & metadata count

---
 R/parser.R | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/R/parser.R b/R/parser.R
index c10c166..c69cd97 100644
--- a/R/parser.R
+++ b/R/parser.R
@@ -303,9 +303,25 @@ parser <- function(file, dec = ".") {
   message(paste("Parsing file:", file))
 
   #number of rows for each part
-  file.lines <- data.table::fread(file, header = FALSE, fill = TRUE, select = 1)[[1]]
-  num.header <- length(grep("^!", file.lines))
-  num.metadata <- length(grep("^#", file.lines))
+  con <- file(file, open = "r")
+  num.header <- 0
+  num.metadata <- 0
+
+  tryCatch(expr = {
+    while(TRUE) {
+      line <- readLines(con = con, n = 1)
+
+      if(grepl("^!", line, perl = TRUE)) {
+        num.header <- num.header + 1
+      } else if(grepl("^#", line, perl = TRUE)) {
+        num.metadata <- num.metadata + 1
+      } else {
+        break
+      }
+    }
+  }, finally = {
+    close(con = con)
+  })
 
   ###parse header
   header <- data.table::fread(input = file, fill = TRUE, header = FALSE, dec = dec, nrows = num.header, integer64 = "double")