A Dataset For Temporal Analysis Of Files Related To The Jfk Case
Markus Luczak–Roesch · Zenodo (CERN European Organization for Nuclear Research) · 2017
This dataset contains the content of the subset of all files with a correct publication date from the 2017 release of files related to the JFK case (retrieved from https://www.archives.gov/research/jfk/2017-release). This content was extracted from the source PDF files using the R OCR libraries tesseract and pdftools. The code to derive the dataset is given as follows: ### BEGIN R DATA PROCESSING SCRIPT library(tesseract) library(pdftools) pdfs <- list.files("/home/STAFF/luczakma/RProjects/JFK/data/files/") meta <- read.csv2("/home/STAFF/luczakma/RProjects/JFK/data/jfkrelease-2017-dce65d0ec70a54d5744de17d280f3ad2.csv",header = T,sep = ',') meta$Doc.Date <- as.character(meta$Doc.Date) meta.clean <- meta[-which(meta$Doc.Date=="" | grepl("/0000",meta$Doc.Date)),] for(i in 1:nrow(meta.clean)){ meta.clean$Doc.Date[i] <- gsub("00","01",meta.clean$Doc.Date[i]) if(nchar(meta.clean$Doc.Date[i])<10){ meta.clean$Doc.Date[i]<-format(strptime(meta.clean$Doc.Date[i],format = "%d/%m/%y"),"%m/%d/%Y") } } meta.clean$Doc.Date <- strptime(meta.clean$Doc.Date,format = "%m/%d/%Y") meta.clean <- meta.clean[order(meta.clean$Doc.Date),] docs <- data.frame(content=character(0),dpub=character(0),stringsAsFactors = F) for(i in 1:nrow(meta.clean)){ #for(i in 1:3){ pdf_prop <- pdftools::pdf_info(paste0("/home/STAFF/luczakma/RProjects/JFK/data/files/",tolower(gsub("\\s+"," ",gsub(" ","",meta.clean$File.Name[i]))))) tmp_files <- c() for(k in 1:pdf_prop$pages){ tmp_files <- c(tmp_files,paste0("/home/STAFF/luczakma/RProjects/JFK/data/tmp/",k)) } img_file <- pdftools::pdf_convert(paste0("/home/STAFF/luczakma/RProjects/JFK/data/files/",tolower(gsub("\\s+"," ",gsub(" ","",meta.clean$File.Name[i])))), format = 'tiff', pages = NULL, dpi = 700,filenames = tmp_files) txt <- "" for(j in 1:length(img_file)){ extract <- ocr(img_file[j], engine = tesseract("eng")) #unlink(img_file) txt <- paste(txt,extract,collapse = " ") } docs <- rbind(docs,data.frame(content=iconv(tolower(gsub("\\s+"," ",gsub("[[:punct:]]|[ ]"," ",txt))),to="UTF-8"),dpub=format(meta.clean$Doc.Date[i],"%Y/%m/%d"),stringsAsFactors = F),stringsAsFactors = F) } ### END R DATA PROCESSING SCRIPT