last update:
2025-01-03
In [30]:
options(repr.plot.width=12, repr.plot.height=6)
suppressPackageStartupMessages(library(dplyr))
suppressPackageStartupMessages(library(xts))
suppressPackageStartupMessages(library(lubridate))
Sys.setlocale("LC_TIME", "C")
'C'
Raw datafiles¶
%% shell script for handling datalogger files ##
cd /home/susana/Work/Field/ENA-ARM/Rawdata/2018_1min/ ls -1v | head -n -4 > filenames.txt
for i in cat filenames.txt ; do {
% remove 1st 4 rows (datalogger header) tail -n +5 $i > tmp.txt
cat TableTMP.txt tmp.txt > rawDatafile_2018_1min.txt rm tmp.txt
cp rawDatafile_2018_1min.txt TableTMP.txt
mv $i dataloggerFiles
} done
rm filenames.txt
Raw datafiles
datafile - 2015 (15 min) [2015-05-07 11:30 - 2015-12-31 23:45]
datafile - 2016 (15 min) [2016-01-01 00:00 - 2016-04-27 23:45]
datafile - 2016 (1 min) [2016-05-04 15:39 - 2016-12-31 23:59]
datafile - 2017 (1 min) [2017-01-01 00:00 - 2017-12-31 23:59]
datafile - 2018 (1 min) [2018-01-01 00:00 - 2018-12-31 23:59]
datafile - 2019 (1 min) [2019-01-01 00:00 - 2019-12-31 23:59]
datafile - 2020 (1 min) [2020-01-01 00:00 - 2020-12-31 23:59]
datafile - 2021 (1 min) [2021-01-01 00:00 - 2021-12-31 23:59]
datafile - 2022 (1 min) [2022-01-01 00:00 - 2022-12-31 23:59]
datafile - 2023 (1 min) [2023-01-01 00:00 - 2023-12-31 23:59]
datafile - 2024 (1 min) [2024-01-01 00:00 - 2024-12-31 23:59]
datafile - 2025 (1 min) [2025-01-01 00:00 - ]
Pre-processing¶
2015 - 15 min¶
In [31]:
dat=read.table(file="datafiles/rawDatafile_2015_15min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
# apply 0.01 multiplier and convert total counts in 15-min to counts per minute (cpm)
counts=dat$V5*100/15
# dataframe of raw variables
#data.only=data.frame(timestamp=new.date.time, gamma=counts,TdegC=dat$V6) #old version
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 15 min measurements --> 60*15=900 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/900)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/900)))!=1)
ngaps
1
** Pre-processing: add missing times as NA values **
In [32]:
library(dplyr)
all.times <- seq.POSIXt(from=as.POSIXct("2015-05-07 11:30:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2015-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [33]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
In [34]:
# plot(x = data.all.times$timestamp,y=data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
Pre-processing: outlier removal
The outlier is a single value (the first measurement after the gap on 2015-12-29 04:30)
In [35]:
data.all.times$gamma[which.min(data.all.times$gamma)]=NA
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
In [36]:
data.all.times=data.frame(format(data.all.times$timestamp),data.all.times$gamma)
In [37]:
# Pre-processed datafile
write.table(file="datafiles/preprocessedDatafile_2015_15min.txt",data.all.times,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [38]:
rm(list = ls()) # clear worspace
2016 - 15 min¶
In [39]:
dat=read.table(file="datafiles/rawDatafile_2016_15min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
#summary(new.date.time)
# apply 0.01 multiplier and convert total counts in 15-min to counts per minute (cpm)
counts=dat$V5*100/15
# dataframe of raw variables
#data.only=data.frame(timestamp=new.date.time, gamma=counts,TdegC=dat$V6) #old version
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 15 min measurements --> 60*15=900 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/900)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/900)))!=1)
ngaps
3
Pre-processing: add missing times as NA values
In [40]:
library(dplyr)
all.times <- seq.POSIXt(from=as.POSIXct("2016-01-01 00:00:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), to=as.POSIXct("2016-04-27 08:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [41]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
In [42]:
#plot(x = data.all.times$timestamp,y=data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
Pre-processing: outlier removal
The outlier is a single value on 2016-01-13, 12:30 (the first measurement after the gap on 2016-01-12 03:00)
In [43]:
data.all.times$gamma[which.min(data.all.times$gamma)]=NA
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
In [44]:
data.all.times=data.frame(format(data.all.times$timestamp),data.all.times$gamma)
In [45]:
## Pre-processed datafile
write.table(file="datafiles/preprocessedDatafile_2016_15min_1.txt",data.all.times,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [46]:
rm(list = ls()) # clear worspace
2016 - 1 min¶
In [47]:
dat=read.table(file="datafiles/rawDatafile_2016_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
#data.only=data.frame(timestamp=new.date.time, gamma=counts,TdegC=dat$V6) # old version
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
ngaps
#data.only$timestamp[502]
0
In [48]:
plot(x = data.only$timestamp,y = data.only$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.only$timestamp,y = data.only$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
In [49]:
data.only=data.frame(format(data.only$timestamp),data.only$gamma)
In [50]:
## Pre-processed datafile (complete days only - since 2016-05-05)
write.table(file="datafiles/preprocessedDatafile_2016_1min.txt",data.only[502:dim(data.only)[1],],sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [51]:
data.only=data.frame(timestamp=new.date.time, gamma=counts)
xts.gamma.1min=xts(data.only[(502:dim(data.only)[1]),]$gamma, order.by=data.only[(502:dim(data.only)[1]),]$timestamp,tzone="UTC")
xts.temp.1min=xts(data.only[(502:dim(data.only)[1]),]$TdegC, order.by=data.only[(502:dim(data.only)[1]),]$timestamp,tzone="UTC")
times.15min <- seq.POSIXt(from=as.POSIXct("2016-05-05 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2016-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
#dat.2016.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1),TdegC=head(as.vector(xts.temp.15min),-1))
dat.2016.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
In [52]:
plot(x = data.only$timestamp,y = data.only$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2016.15min$timestamp,y=dat.2016.15min$gamma,col=4)
In [53]:
#plot(x = data.only$timestamp,y = data.only$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
#lines(x=dat.2016.15min$timestamp,y=dat.2016.15min$TdegC,col=4)
In [54]:
out=data.frame(format(dat.2016.15min$timestamp), dat.2016.15min$gamma)
In [55]:
## Pre-processed datafile (2016, from direct 15min measurements and from 1-min aggregation)
write.table(file="datafiles/preprocessedDatafile_2016_15min_2.txt",out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [56]:
rm(list = ls()) # clear worspace
2017 - 1 min¶
In [57]:
dat=read.table(file="datafiles/rawDatafile_2017_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
summary(new.date.time)
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
#data.only=data.frame(timestamp=new.date.time, gamma=counts,TdegC=dat$V6) # old version
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
ngaps
Min. 1st Qu.
"2017-01-01 00:00:00.0000" "2017-04-02 06:00:15.0000"
Median Mean
"2017-07-02 12:00:30.0000" "2017-07-02 12:01:07.5763"
3rd Qu. Max.
"2017-10-01 18:02:45.0000" "2017-12-31 23:59:00.0000"
5
Pre-processing: add missing times as NA values
In [58]:
library(dplyr)
all.times <- seq.POSIXt(from=as.POSIXct("2017-01-01 00:00:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2017-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [59]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
Pre-processing: outlier removal
Outliers (set as NA):
- single low value on 2017-05-23 (first measurement after the gap on 2017-05-23 14:01)
- single low value on 2017-05-27 (first measurement after the gap on 2017-05-27 09:59)
- single low value on 2017-07-19 07:48
The data from 5 July 2017 [10:00 to 17:50] are also set as NA as related to tests and displacement of sensor field work
In [60]:
library(lubridate)
# data from testing period on July 5th set as NA
date1 <- as.POSIXct("2017-07-05 10:00:00",tz="UTC")
date2 <- as.POSIXct("2017-07-05 17:59:00",tz="UTC")
test.dates <- interval(date1, date2)
data.all.times[data.all.times$timestamp %within% test.dates,2]=NA
#data.all.times[data.all.times$timestamp %within% test.dates,3]=NA
out1 <- as.POSIXct("2017-05-23 14:01:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
out2 <- as.POSIXct("2017-05-27 10:00:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
out3 <- as.POSIXct("2017-07-19 07:48:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
data.all.times[which(data.all.times$timestamp == out1),2]=NA
data.all.times[which(data.all.times$timestamp == out2),2]=NA
data.all.times[which(data.all.times$timestamp == out3),2]=NA
In [61]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
In [62]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- as.POSIXct("2017-01-01 00:00:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
last.time <- as.POSIXct("2017-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2017 <- interval(first.time,last.time)
dat1min=data.all.times[data.all.times$timestamp %within% complete.2017,]
out=data.frame(format(dat1min$timestamp), dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2017_1min.txt",out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [63]:
times.15min <- seq.POSIXt(from=as.POSIXct("2017-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2017-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
library(xts)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
dat.2017.15min=data.frame(timestamp=format(times.15min), gamma=head(as.vector(xts.gamma.15min),-1))
In [64]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2017.15min$timestamp,y=dat.2017.15min$gamma,col=4)
Warning message in xy.coords(x, y): “NAs introduced by coercion”
In [65]:
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
#lines(x=dat.2017.15min$timestamp,y=dat.2017.15min$TdegC,col=4)
In [66]:
## Pre-processed datafile
write.table(file="datafiles/preprocessedDatafile_2017_15min.txt",dat.2017.15min,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [67]:
rm(list = ls()) # clear worspace
2018 - 1 min¶
In [68]:
dat=read.table(file="datafiles/rawDatafile_2018_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
start.time=new.date.time[1]
end.time=new.date.time[length(new.date.time)]
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
#data.only=data.frame(timestamp=new.date.time, gamma=counts,TdegC=dat$V6) #old version
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
start.time
end.time
ngaps
[1] "2018-01-01 UTC"
[1] "2018-12-31 23:59:00 UTC"
3
Pre-processing: add missing times as NA values
In [69]:
library(dplyr)
all.times <- seq.POSIXt(from=start.time, to=end.time, by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [70]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
Pre-processing: outlier removal
- gamma counts below 6500 set as missing
(typically after power loss, the last and first value after a gap is too low)
In [71]:
data.all.times$gamma[which(data.all.times$gamma < 6500)]=NA
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
In [72]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- start.time
last.time <- as.POSIXct("2018-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2018 <- interval(first.time,last.time)
dat1min=data.all.times[data.all.times$timestamp %within% complete.2018,]
out=data.frame(format(dat1min$timestamp),dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2018_1min.txt",out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [73]:
times.15min <- seq.POSIXt(from=as.POSIXct("2018-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2018-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
library(xts)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
dat.2018.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
In [74]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2018.15min$timestamp,y=dat.2018.15min$gamma,col=4)
In [75]:
out=data.frame(format(dat.2018.15min$timestamp),dat.2018.15min$gamma)
## Pre-processed datafile
write.table(file="datafiles/preprocessedDatafile_2018_15min.txt",out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [76]:
rm(list = ls()) # clear workspace
2019 - 1 min¶
In [77]:
dat=read.table(file="datafiles/rawDatafile_2019_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
start.time=new.date.time[1]
end.time=new.date.time[length(new.date.time)]
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
start.time
end.time
ngaps
gaps
[1] "2019-01-01 UTC"
[1] "2019-12-31 23:59:00 UTC"
1
447021
In [78]:
## Outlier value on "2019-10-30 13:54:00" set as NA (likely as a result of a short power outage)
counts[which.min(counts)]=NA
data.only=data.frame(timestamp=new.date.time, gamma=counts)
Pre-processing: add missing times as NA values
In [79]:
library(dplyr)
all.times <- seq.POSIXt(from=start.time, to=end.time, by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [80]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
In [81]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- start.time
last.time <- as.POSIXct("2019-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2019 <- interval(first.time,last.time)
dat1min=data.all.times[data.all.times$timestamp %within% complete.2019,]
out=data.frame(format(dat1min$timestamp),dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2019_1min.txt",out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [82]:
times.15min <- seq.POSIXt(from=as.POSIXct("2019-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2019-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
xts.gamma.15min[29049]=NA
xts.gamma.15min[29050]=NA
xts.gamma.15min[29051]=NA
xts.gamma.15min[29052]=NA
xts.gamma.15min[29053]=NA
dat.2019.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
out=data.frame(format(dat.2019.15min$timestamp),dat.2019.15min$gamma )
In [83]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2019.15min$timestamp,y=dat.2019.15min$gamma,col=4)
In [84]:
## Pre-processed datafile
write.table(file="datafiles/preprocessedDatafile_2019_15min.txt",out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [85]:
rm(list = ls()) # clear workspace
2020 - 1 min¶
In [86]:
dat=read.table(file="datafiles/rawDatafile_2020_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE,fill=TRUE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
start.time=new.date.time[1]
end.time=new.date.time[length(new.date.time)]
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
start.time
end.time
ngaps
gaps
[1] "2020-01-01 UTC"
[1] "2020-12-31 23:59:00 UTC"
2
- 383901
- 383911
In [87]:
366*24*60
366*24*4
527040
35136
In [88]:
## Outlier values below 6500 after '2020-09-23 13:53:00'" set as NA
# (likely as a result of a short power outage)
counts[which(counts < 6500)]=NA
data.only=data.frame(timestamp=new.date.time, gamma=counts)
Pre-processing: add missing times as NA values
In [89]:
library(dplyr)
all.times <- seq.POSIXt(from=start.time, to=end.time, by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [90]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
In [91]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- start.time
last.time <- as.POSIXct("2020-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2020 <- interval(first.time,last.time)
dat1min=data.all.times[data.all.times$timestamp %within% complete.2020,]
out=data.frame(format(dat1min$timestamp),dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2020_1min.txt",out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [92]:
times.15min <- seq.POSIXt(from=as.POSIXct("2020-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2020-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
dat.2020.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
out=data.frame(format(dat.2020.15min$timestamp),dat.2020.15min$gamma)
In [93]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2020.15min$timestamp,y=dat.2020.15min$gamma,col=4)
In [94]:
## Pre-processed datafile
write.table(file="datafiles/preprocessedDatafile_2020_15min.txt",out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [95]:
rm(list = ls()) # clear workspace
2021 - 1 min¶
In [96]:
dat=read.table(file="datafiles/rawDatafile_2021_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE,fill=TRUE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
start.time=new.date.time[1]
end.time=new.date.time[length(new.date.time)]
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
start.time
end.time
ngaps
[1] "2021-01-01 UTC"
[1] "2021-12-31 23:59:00 UTC"
0
In [97]:
## Outlier values below 6500 after '2020-09-23 13:53:00'" set as NA
# (likely as a result of a short power outage)
#counts[which(counts < 6500)]=NA
#data.only=data.frame(timestamp=new.date.time, gamma=counts,TdegC=dat$V6)
Pre-processing: add missing times as NA values
In [98]:
library(dplyr)
all.times <- seq.POSIXt(from=start.time, to=end.time, by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [99]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
In [100]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- start.time
last.time <- as.POSIXct("2021-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2021 <- interval(first.time,last.time)
dat1min=data.all.times[data.all.times$timestamp %within% complete.2021,]
out=data.frame(format(dat1min$timestamp),dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2021_1min.txt",out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [101]:
times.15min <- seq.POSIXt(from=as.POSIXct("2021-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2021-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
dat.2021.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
out=data.frame(format(dat.2021.15min$timestamp),dat.2021.15min$gamma)
In [102]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2021.15min$timestamp,y=dat.2021.15min$gamma,col=4,type="b",pch=20)
In [103]:
## Pre-processed datafile
write.table(file="datafiles/preprocessedDatafile_2021_15min.txt",out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [104]:
rm(list = ls()) # clear workspace
2022 - 1 min¶
In [105]:
dat=read.table(file="datafiles/rawDatafile_2022_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE,fill=TRUE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
start.time=new.date.time[1]
end.time=new.date.time[length(new.date.time)]
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
start.time
end.time
[1] "2022-01-01 UTC"
[1] "2022-12-31 23:59:00 UTC"
Pre-processing: add missing times as NA values
In [106]:
all.times <- seq.POSIXt(from=start.time, to=end.time, by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [107]:
gamma.xts=xts(data.all.times$gamma,data.all.times$timestamp)
plot(gamma.xts,main="")
plot(gamma.xts["2022-05-17/"], type="b",pch=20,main="")
#plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
Pre-processing: set suspicious values as NA
Gamma values from 2022-05-17 to 2022-07-18 are suspicious and set as missing (NA).
Feedback from the station technicians: unusual human activity in the container where the sensor is located, sensor was not in its usual up-right position for some time
In [108]:
gamma.xts["2022-05-17 00:00:00/2022-07-18 23:59:00"]=NA
data.all.times=data.frame(timestamp=index(gamma.xts), gamma=as.vector(gamma.xts))
In [109]:
plot(as.zoo(gamma.xts),main="")
In [110]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- start.time
last.time <- as.POSIXct("2022-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2022 <- interval(first.time,last.time)
dat1min=data.all.times[data.all.times$timestamp %within% complete.2022,]
out=data.frame(format(dat1min$timestamp),dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2022_1min.txt",out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [111]:
times.15min <- seq.POSIXt(from=as.POSIXct("2022-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2022-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum,na.rm=TRUE)
dat.2022.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
In [112]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2022.15min$timestamp,y=dat.2022.15min$gamma,col=4,type="b",pch=20)
In [113]:
dat.2022.15min$gamma[which(dat.2022.15min$gamma<6500)]=NA
In [114]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2022.15min$timestamp,y=dat.2022.15min$gamma,col=4,type="b",pch=20)
In [115]:
## Pre-processed datafile
out=data.frame(format(dat.2022.15min$timestamp),dat.2022.15min$gamma)
write.table(file="datafiles/preprocessedDatafile_2022_15min.txt",out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [116]:
rm(list = ls()) # clear workspace
2023 - 1 min¶
In [117]:
dat=read.table(file="datafiles/rawDatafile_2023_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE,fill=TRUE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
start.time=new.date.time[1]
end.time=new.date.time[length(new.date.time)]
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
start.time
end.time
ngaps
[1] "2023-01-01 UTC"
[1] "2023-12-31 23:59:00 UTC"
0
Pre-processing: add missing times as NA values
In [118]:
library(dplyr)
all.times <- seq.POSIXt(from=start.time, to=end.time, by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [119]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
#plot(x = data.all.times$timestamp,y = data.all.times$TdegC,type="b",pch=20,xlab="time",ylab="T (degC)")
In [120]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- start.time
last.time <- as.POSIXct("2023-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2023 <- interval(first.time,last.time)
dat1min=data.all.times[data.all.times$timestamp %within% complete.2023,]
out=data.frame(format(dat1min$timestamp),dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2023_1min.txt",out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [121]:
times.15min <- seq.POSIXt(from=as.POSIXct("2023-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2023-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
dat.2023.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
In [122]:
plot(x = data.all.times$timestamp,y = data.all.times$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
lines(x=dat.2023.15min$timestamp,y=dat.2023.15min$gamma,col=4,type="b",pch=20)
In [123]:
## Pre-processed datafile
out=data.frame(format(dat.2023.15min$timestamp),dat.2023.15min$gamma)
write.table(file="datafiles/preprocessedDatafile_2023_15min.txt",out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [124]:
rm(list = ls()) # clear workspace
2024 - 1 min¶
In [125]:
dat=read.table(file="datafiles/rawDatafile_2024_1min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE,fill=TRUE)
# set date/time as POSIXct
new.date.time=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
start.time=new.date.time[1]
end.time=new.date.time[length(new.date.time)]
# apply 0.01 multiplier
counts=dat$V5*100
# dataframe of raw variables
data.only=data.frame(timestamp=new.date.time, gamma=counts)
# check for continuity of record [0 --> continuous record]; ngaps=number of gaps
# 1 min measurements --> 60 sec
ngaps=length(which(c(1,round(diff(unclass(new.date.time)/60)))!=1))
gaps=which(c(1,round(diff(unclass(new.date.time)/60)))!=1)
start.time
end.time
ngaps
[1] "2024-01-01 UTC"
[1] "2024-12-31 23:59:00 UTC"
0
Pre-processing: add missing times as NA values
In [126]:
library(dplyr)
all.times <- seq.POSIXt(from=start.time, to=end.time, by=60)
continuous.time=as.POSIXct(format.POSIXct(all.times),format="%Y-%m-%d %H:%M:%S",tz="UTC")
df <- data.frame(timestamp=continuous.time,stringsAsFactors=FALSE)
data.all.times <- full_join(df,data.only)
Joining with `by = join_by(timestamp)`
In [127]:
plot(x = data.only$timestamp,y = data.only$gamma,type="b",pch=20,xlab="time",ylab="counts (cpm)")
In [128]:
## Pre-processed datafile (complete days only - up to last.time )
first.time <- start.time
last.time <- as.POSIXct("2024-12-31 23:59:00",format="%Y-%m-%d %H:%M:%S",tz="UTC")
complete.2024 <- interval(first.time,last.time)
dat1min=data.all.times[data.only$timestamp %within% complete.2024,]
In [129]:
dat1min.out=data.frame(format(dat1min$timestamp),dat1min$gamma)
write.table(file="datafiles/preprocessedDatafile_2024_1min.txt",dat1min.out,quote=FALSE,row.names=FALSE,col.names=FALSE)
Pre-processing: aggregation to 15 min
In [130]:
times.15min <- seq.POSIXt(from=as.POSIXct("2024-01-01 00:15:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2024-12-31 23:45:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.15min=period.apply(x=xts.gamma.1min/15, INDEX=endpoints(xts.gamma.1min, on="mins", k=15), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
dat.2024.15min=data.frame(timestamp=times.15min, gamma=head(as.vector(xts.gamma.15min),-1))
Pre-processing: aggregation to 60 min
In [131]:
times.60min <- seq.POSIXt(from=as.POSIXct("2024-01-01 00:00:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"),
to=as.POSIXct("2024-12-31 23:00:00",format="%Y-%m-%d %H:%M:%S",tz="UTC"), by=900*4)
xts.gamma.1min=xts(dat1min$gamma, order.by=dat1min$timestamp,tzone="UTC")
#xts.temp.1min=xts(dat1min$TdegC , order.by=dat1min$timestamp,tzone="UTC")
xts.gamma.60min=period.apply(x=xts.gamma.1min/60, INDEX=endpoints(xts.gamma.1min, on="mins", k=60), FUN=sum)
#xts.temp.15min=period.apply(x=xts.temp.1min, INDEX=endpoints(xts.temp.1min, on="mins", k=15), FUN=mean)
dat.2024.60min=data.frame(timestamp=times.60min, gamma=as.vector(xts.gamma.60min))
In [132]:
#pdf(file="gamma.pdf",paper="a4r",width=9,height=5)
plot(x = data.only$timestamp,y = data.only$gamma,type="b",pch=20,xlab="Time",ylab="Gamma counts per minute", main=2024,col="darkgrey")
lines(x=dat.2024.15min$timestamp,y=dat.2024.15min$gamma,col=4,type="l",pch=20)
lines(x=dat.2024.60min$timestamp,y=dat.2024.60min$gamma,col="darkblue",type="l",pch=20)
#dev.off()
In [133]:
## Pre-processed datafile
dat.2024.15min.out=data.frame(format(dat.2024.15min$timestamp),dat.2024.15min$gamma)
write.table(file="datafiles/preprocessedDatafile_2024_15min.txt",dat.2024.15min.out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [134]:
## Pre-processed datafile
dat.2024.60min.out=data.frame(format(dat.2024.60min$timestamp),dat.2024.60min$gamma)
write.table(file="datafiles/preprocessedDatafile_2024_hourly.txt",dat.2024.60min.out,sep=",",quote=FALSE,row.names=FALSE,col.names=FALSE)
In [135]:
rm(list = ls()) # clear workspace
2025 - 1 min¶
In [ ]:
In [136]:
rm(list = ls()) # clear workspace
Pre-processed datafiles¶
datafile - 2015 (15 min) [2015-05-07 11:30 - 2015-12-31 23:45]
datafile - 2016 (15 min) [2016-01-01 00:00 - 2016-04-27 08:15]
datafile - 2016 (1 min) [2016-05-05 00:00 - 2016-12-31 23:59]
datafile - 2016 (15 min) [2016-05-05 00:15 - 2016-12-31 23:45]
datafile - 2017 (1 min) [2017-01-01 00:00 - 2017-12-31 23:59 ]
datafile - 2017 (15 min) [2017-01-01 00:00 - 2017-07-25 23:45 ]
datafile - 2018 (15 min) [2018-01-01 00:00 - 2018-12-31 23:45]
datafile - 2019 (15 min) [2019-01-01 00:00 - 2019-12-31 23:45]
datafile - 2020 (15 min) [2020-01-01 00:00 - 2020-12-31 23:45]
datafile - 2021 (15 min) [2021-01-01 00:00 - 2021-12-31 23:45]
datafile - 2022 (15 min) [2022-01-01 00:00 - 2022-12-31 23:45]
datafile - 2023 (15 min) [2023-01-01 00:00 - 2023-12-31 23:45]
datafile - 2024 (15 min) [2024-01-01 00:00 - 2024-12-31 23:45]
%% shell script for joining pre-processed datafiles
cat preprocessedDatafile_2015_15min.txt preprocessedDatafile_2016_15min_1.txt preprocessedDatafile_2016_15min_2.txt preprocessedDatafile_2017_15min.txt preprocessedDatafile_2018_15min.txt preprocessedDatafile_2019_15min.txt preprocessedDatafile_2020_15min.txt preprocessedDatafile_2021_15min.txt preprocessedDatafile_2022_15min.txt preprocessedDatafile_2023_15min.txt preprocessedDatafile_2024_15min.txt > ENAgamma_15min.txt
%% shell script for joining pre-processed datafiles
cat preprocessedDatafile_2016_1min.txt preprocessedDatafile_2017_1min.txt preprocessedDatafile_2018_1min.txt preprocessedDatafile_2019_1min.txt preprocessedDatafile_2020_1min.txt preprocessedDatafile_2021_1min.txt preprocessedDatafile_2022_1min.txt preprocessedDatafile_2023_1min.txt preprocessedDatafile_2024_1min.txt > ENAgamma_1min.txt
Pre-processed time series¶
In [137]:
dat=read.table(file="datafiles/ENAgamma_15min.txt",header=FALSE,sep=",",stringsAsFactors=FALSE)
# data date/time as POSIXct
dat.times15=as.POSIXct(dat$V1,format="%Y-%m-%d %H:%M:%S",tz="UTC")
gamma15min.xts=xts(dat$V2,dat.times15)
anyDuplicated(index(gamma15min.xts))
0
In [138]:
plot(as.zoo(gamma15min.xts),xlab="Time",ylab="counts/minute")
In [ ]: