Skip to content
Snippets Groups Projects
Commit 38d03122 authored by Adriaens, Ines's avatar Adriaens, Ines :v_tone2:
Browse files

documentation of tools

parent 30807ab4
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
behaviours_percow.py
+ 187
− 193
View file @ 38d03122
...@@ -5,14 +5,6 @@ Created on Tue Feb 14 12:08:41 2023 ...@@ -5,14 +5,6 @@ Created on Tue Feb 14 12:08:41 2023
@author: adria036 @author: adria036
""" """
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 8 16:47:19 2023
@author: adria036
"""
import os import os
os.chdir(r"C:\Users\adria036\OneDrive - Wageningen University & Research\iAdriaens_doc\Projects\cKamphuis\nlas\scripts\uwb") os.chdir(r"C:\Users\adria036\OneDrive - Wageningen University & Research\iAdriaens_doc\Projects\cKamphuis\nlas\scripts\uwb")
...@@ -22,9 +14,6 @@ os.chdir(r"C:\Users\adria036\OneDrive - Wageningen University & Research\iAdriae ...@@ -22,9 +14,6 @@ os.chdir(r"C:\Users\adria036\OneDrive - Wageningen University & Research\iAdriae
from datetime import date, datetime from datetime import date, datetime
import pandas as pd import pandas as pd
import numpy as np import numpy as np
# import matplotlib.pyplot as plt
# import matplotlib
# import seaborn as sns
from uwbfunctions import behaviour_bouts from uwbfunctions import behaviour_bouts
#%matplotlib qt #%matplotlib qt
...@@ -45,188 +34,193 @@ del fna, path_zones ...@@ -45,188 +34,193 @@ del fna, path_zones
# settings # settings
settings = {'barn' : [60,61,62,70,71,72,73], for barnno in [71,72,73]:
'startdate' : date(2021,11,1), settings = {'barn' : [barnno],
'enddate' : date(2022,2,15), 'startdate' : date(2021,11,1),
'cows' : 0, # or specific cow number 'enddate' : date(2023,2,15),
} 'cows' : 0, # or specific cow number
}
# files that comply with settings
fn = []
for b in range(0,len(settings["barn"])):
print("barn = " + str(settings["barn"][b]))
if settings["cows"] == 0:
fbarn = [f for f in os.listdir(path + "/barn" + str(settings["barn"][b])) \
if os.path.isfile(os.path.join(path,"barn"+str(settings["barn"][b]),f)) \
and (datetime.strptime(f[5:13], '%Y%m%d').date() >= settings["startdate"]) \
and (datetime.strptime(f[5:13], '%Y%m%d').date() <= settings["enddate"])]
fbarn.sort()
else:
fbarn = [f for f in os.listdir(path + "/barn" + str(settings["barn"][b])) \
if os.path.isfile(os.path.join(path,"barn"+str(settings["barn"][b]),f)) \
and (int(f[26:-4]) in settings["cows"]) \
and (datetime.strptime(f[5:13], '%Y%m%d').date() >= settings["startdate"]) \
and (datetime.strptime(f[5:13], '%Y%m%d').date() <= settings["enddate"])]
fbarn.sort()
fn.extend(fbarn)
fn.sort()
# find unique cows
cows = list(set([int(f[26:-4]) for f in fn]))
# read data and calculate bouts and behaviours
for cow in cows:
print("read data of cow " + str(cow))
# read data
data = pd.DataFrame([])
for f in fn:
if str(cow) in f:
print(f)
barn = f[19:21]
sub = pd.read_csv(path + "/barn" + barn + "/" + f,
usecols = ["cowid","barn","date","t","xnew","ynew","area","zone","X","y"],
dtype = {"cowid" : "int64","barn" : "int64","date" : "object",
"t" : "int64", "xnew":"float64","ynew":"float64",
"area":"object","zone":"float64","X":"float64","y" : "float64"})
sub["date"] = pd.to_datetime(sub["date"], format = "%Y-%m-%d")
data = pd.concat([data,sub])
data = data.sort_values(by = ["cowid","date","t"])
data = data.reset_index(drop=1)
data.loc[data["zone"].isna(),"zone"] = 8
data.loc[data["zone"] == 8 ,"area"] = "unknown"
data["date"] = data["date"].dt.date
# calculate and summarize behaviours
bouts = pd.DataFrame([])
bsums = pd.DataFrame([])
act_dist = pd.DataFrame([])
# summarize days # files that comply with settings
days = data.loc[(data["cowid"]==cow),["date"]].drop_duplicates() fn = []
days = days.sort_values(by = ["date"]).reset_index(drop=1) for b in range(0,len(settings["barn"])):
print("barn = " + str(settings["barn"][b]))
# calculate bouts and active behaviours if settings["cows"] == 0:
for dd in days["date"]: fbarn = [f for f in os.listdir(path + "/barn" + str(settings["barn"][b])) \
print("cow = " + str(cow) + ", date = " + str(dd)) if os.path.isfile(os.path.join(path,"barn"+str(settings["barn"][b]),f)) \
dset = data.loc[(data["cowid"] == cow) & (data["date"] == dd)].copy() and (datetime.strptime(f[5:13], '%Y%m%d').date() >= settings["startdate"]) \
and (datetime.strptime(f[5:13], '%Y%m%d').date() <= settings["enddate"])]
# ------------------------ feeding behaviour -------------------------- fbarn.sort()
interval = 5*60 # min number of seconds between separate bouts else:
behaviour = "feed" # feeding behaviour fbarn = [f for f in os.listdir(path + "\\barn" + str(settings["barn"][b])) \
min_length = 30 # do not count if shorter than 30 seconds if os.path.isfile(os.path.join(path,"barn"+str(settings["barn"][b]),f)) \
df,summary = behaviour_bouts(dset, interval, behaviour,min_length) and (int(f[26:-4]) in settings["cows"]) \
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1) and (datetime.strptime(f[5:13], '%Y%m%d').date() >= settings["startdate"]) \
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1) and (datetime.strptime(f[5:13], '%Y%m%d').date() <= settings["enddate"])]
fbarn.sort()
# ---------------------- drinking behaviour --------------------------- fn.extend(fbarn)
interval = 5*60 # min number of seconds between separate bouts fn.sort()
behaviour = "drink" # drinking behaviour
min_length = 5 # do not count if shorter than 2 seconds # find unique cows
df,summary = behaviour_bouts(dset, interval, behaviour, min_length) cows = list(set([int(f[26:-4]) for f in fn]))
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1) cows.sort()
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1) tel=0
# read data and calculate bouts and behaviours
# ---------------------- resting behaviour ---------------------------- for cow in cows:
interval = 2*60 # min number of seconds between separate bouts tel += 1
behaviour = "cubicle" # resting behaviour print("read data of cow " + str(cow) + ", this is cow " + str(tel) + " out of " + str(len(cows)))
min_length = 30 # do not count if shorter than 30 seconds # read data
df,summary = behaviour_bouts(dset, interval, behaviour, min_length) data = pd.DataFrame([])
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1) for f in fn:
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1) if "cow_" + str(cow) in f:
print(f)
# -------------------------- cubicle_A -------------------------------- barn = f[19:21]
interval = 2*60 # min number of seconds between separate bouts sub = pd.read_csv(path + "/barn" + barn + "/" + f,
behaviour = "cubicle_A" # resting behaviour usecols = ["cowid","barn","date","t","xnew","ynew","area","zone","X","y"],
min_length = 30 # do not count if shorter than 30 seconds dtype = {"cowid" : "int64","barn" : "int64","date" : "object",
df,summary = behaviour_bouts(dset, interval, behaviour, min_length) "t" : "int64", "xnew":"float64","ynew":"float64",
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1) "area":"object","zone":"float64","X":"float64","y" : "float64"})
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1) sub["date"] = pd.to_datetime(sub["date"], format = "%Y-%m-%d")
data = pd.concat([data,sub])
# -------------------------- cubicle_B -------------------------------- data = data.sort_values(by = ["cowid","date","t"])
interval = 2*60 # min number of seconds between separate bouts data = data.reset_index(drop=1)
behaviour = "cubicle_B" # resting behaviour
min_length = 30 # do not count if shorter than 30 seconds data.loc[data["zone"].isna(),"zone"] = 8
df,summary = behaviour_bouts(dset, interval, behaviour, min_length) data.loc[data["zone"] == 8 ,"area"] = "unknown"
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1) data["date"] = data["date"].dt.date
# -------------------------- cublicle_C ------------------------------- # calculate and summarize behaviours
interval = 2*60 # min number of seconds between separate bouts bouts = pd.DataFrame([])
behaviour = "cubicle_C" # resting behaviour bsums = pd.DataFrame([])
min_length = 30 # do not count if shorter than 30 seconds act_dist = pd.DataFrame([])
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------ concentrate feeder -------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "concentrate" # concentrate feeding behaviour
min_length = 0 # always count
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------ waiting area ------------------------------
interval = 20*60 # min number of seconds between separate bouts
behaviour = "wait" # in the waiting area
min_length = 0 # always count
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------ unknown area ------------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "unknown" # unknown behaviour
min_length = 10 # do not count if shorter than 10 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------- slatted (barn 62) -------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "slatted" # unknown behaviour
min_length = 10 # do not count if shorter than 10 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------- resting (barn 62) -------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "resting" # unknown behaviour
min_length = 10 # do not count if shorter than 10 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# # ------------------------- distance travelled ------------------------
dset.loc[:,"dist"] = 0
dset.iloc[1:,dset.columns == "dist"] = \
np.sqrt((dset["xnew"].iloc[:-1].values-dset["xnew"].iloc[1:].values)**2 + \
(dset["ynew"].iloc[:-1].values-dset["ynew"].iloc[1:].values)**2)
dset.loc[(dset["dist"]<0.30) | (dset["dist"]>5),"dist"] = 0
# distance travelled and % active (with and without nans)
act = dset[["cowid","date","barn","dist"]].groupby(by = ["cowid","date","barn"]).sum().reset_index()
act["dist"] = round(act["dist"],2)
act["perc_act"] = round(len(dset.loc[dset["dist"] > 0])/len(dset)*100,2)
act["perc_act_nan"] = round(len(dset.loc[dset["dist"] > 0])/len(dset.loc[dset["area"]!="unknown"])*100,2)
# % of the day not in the cubicles
act["perc_no_lying"] = 100- round(len(dset.loc[(dset["area"].str.contains("cubicle") == True) | \
(dset["area"].str.contains("resting") == True)])/len(dset)*100,2)
act["perc_no_lying_nan"] = 100- round(len(dset.loc[(dset["area"].str.contains("cubicle") == True) | \
(dset["area"].str.contains("resting") == True)])/len(dset.loc[dset["area"]!="unknown"])*100,2)
act_dist = pd.concat([act_dist,act]).reset_index(drop=1) # summarize days
days = data.loc[(data["cowid"]==cow),["date"]].drop_duplicates()
days = days.sort_values(by = ["date"]).reset_index(drop=1)
# calculate bouts and active behaviours
for dd in days["date"]:
# print("cow = " + str(cow) + ", date = " + str(dd))
dset = data.loc[(data["cowid"] == cow) & (data["date"] == dd)].copy()
# ------------------------ feeding behaviour --------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "feed" # feeding behaviour
min_length = 30 # do not count if shorter than 30 seconds
df,summary = behaviour_bouts(dset, interval, behaviour,min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ---------------------- drinking behaviour ---------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "drink" # drinking behaviour
min_length = 5 # do not count if shorter than 2 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ---------------------- resting behaviour ----------------------------
interval = 2*60 # min number of seconds between separate bouts
behaviour = "cubicle" # resting behaviour
min_length = 30 # do not count if shorter than 30 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# -------------------------- cubicle_A --------------------------------
interval = 2*60 # min number of seconds between separate bouts
behaviour = "cubicle_A" # resting behaviour
min_length = 30 # do not count if shorter than 30 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# -------------------------- cubicle_B --------------------------------
interval = 2*60 # min number of seconds between separate bouts
behaviour = "cubicle_B" # resting behaviour
min_length = 30 # do not count if shorter than 30 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# -------------------------- cublicle_C -------------------------------
interval = 2*60 # min number of seconds between separate bouts
behaviour = "cubicle_C" # resting behaviour
min_length = 30 # do not count if shorter than 30 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------ concentrate feeder -------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "concentrate" # concentrate feeding behaviour
min_length = 0 # always count
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------ waiting area ------------------------------
interval = 20*60 # min number of seconds between separate bouts
behaviour = "wait" # in the waiting area
min_length = 0 # always count
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------ unknown area ------------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "unknown" # unknown behaviour
min_length = 10 # do not count if shorter than 10 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------- slatted (barn 62) -------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "slatted" # unknown behaviour
min_length = 10 # do not count if shorter than 10 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# ------------------------- resting (barn 62) -------------------------
interval = 5*60 # min number of seconds between separate bouts
behaviour = "resting" # unknown behaviour
min_length = 10 # do not count if shorter than 10 seconds
df,summary = behaviour_bouts(dset, interval, behaviour, min_length)
bouts = pd.concat([bouts,df],axis = 0).reset_index(drop=1)
bsums = pd.concat([bsums,summary],axis = 0).reset_index(drop=1)
# # ------------------------- distance travelled ------------------------
dset.loc[:,"dist"] = 0
dset.iloc[1:,dset.columns == "dist"] = \
np.sqrt((dset["xnew"].iloc[:-1].values-dset["xnew"].iloc[1:].values)**2 + \
(dset["ynew"].iloc[:-1].values-dset["ynew"].iloc[1:].values)**2)
dset.loc[(dset["dist"]<0.30) | (dset["dist"]>5),"dist"] = 0
# distance travelled and % active (with and without nans)
act = dset[["cowid","date","barn","dist"]].groupby(by = ["cowid","date","barn"]).sum().reset_index()
act["dist"] = round(act["dist"],2)
act["perc_act"] = round(len(dset.loc[dset["dist"] > 0])/len(dset)*100,2)
act["perc_act_nan"] = round(len(dset.loc[dset["dist"] > 0])/(0.0001+len(dset.loc[dset["area"]!="unknown"]))*100,2)
# % of the day not in the cubicles
act["perc_no_lying"] = 100- round(len(dset.loc[(dset["area"].str.contains("cubicle") == True) | \
(dset["area"].str.contains("resting") == True)])/len(dset)*100,2)
act["perc_no_lying_nan"] = 100- round(len(dset.loc[(dset["area"].str.contains("cubicle") == True) | \
(dset["area"].str.contains("resting") == True)])/(0.0001+len(dset.loc[dset["area"]!="unknown"]))*100,2)
act_dist = pd.concat([act_dist,act]).reset_index(drop=1)
# save bouts, summaries and behaviours per cow id
act_dist["dist"] = round(act_dist["dist"],2)
act_dist["perc_no_lying_nan"] = round(act_dist["perc_no_lying_nan"],2)
act_dist["perc_no_lying"] = round(act_dist["perc_no_lying"],2)
act_dist.to_csv(svpath + "/activity_cow_" + str(cow) + "_barn" + str(barn) + ".txt")
bouts.to_csv(svpath + "/bouts_cow_" + str(cow) + "_barn" + str(barn) + ".txt")
bsums["total2"] = round(bsums["total2"],2)
bsums["total1"] = round(bsums["total1"],2)
bsums.to_csv(svpath + "/summary_cow_" + str(cow) + "_barn" + str(barn) + ".txt")
# save bouts, summaries and behaviours per cow id del data, act_dist, bouts, bsums, interval, dset, df, summary, min_length
act_dist["dist"] = round(act_dist["dist"],2)
act_dist["perc_no_lying_nan"] = round(act_dist["perc_no_lying_nan"],2)
act_dist["perc_no_lying"] = round(act_dist["perc_no_lying"],2)
act_dist.to_csv(svpath + "/activity_cow_" + str(cow) + ".txt")
bouts.to_csv(svpath + "/bouts_cow_" + str(cow) + ".txt")
bsums["total2"] = round(bsums["total2"],2)
bsums["total1"] = round(bsums["total1"],2)
bsums.to_csv(svpath + "/summary_cow_" + str(cow) + ".txt")
documentation/sensmap_visualisation.png 0 → 100644
+ 0
− 0
View file @ 38d03122
documentation/sensmap_visualisation.png

271 KiB

nlas_tools.md 0 → 100644
+ 34
− 0
View file @ 38d03122
## NLAS tools
__*created by:*__ Ines Adriaens - adria036
__*created on:*__ 16/02/2023
__*collaborators:*__ Bert Klandermans
### Context
NLAS aims at developing tools for the next generation of animal science. In our case, *"locomotion of dairy cows"*, we developed technology to
track and monitor locomotion of dairy cows in a continuous and automated way. We worked with 2 different technologies: a UWB-based positioning
system, and video analyses.
This document lists and explains the tools that are developed in the context of NLAS using the UWB data collected at Dairy Campus, Leeuwarden.
Most tools described below consist of software code to collect, save, quality-control, preprocess and interpret the positioning data.
### Description of the sensor system and setup
A real-time UWB positioning system was installed at Dairy Campus, Leeuwarden, the Netherlands. It consists of antennas (at least 4 per barn), and tags that
are mounted on the top-side of the neck of the cows. Only when a tag connects to all 4 antennas, the sensor deems to have a reliable signal and the location is
calculated, with an estimated accuracy of 30 cm.
The initial installation was done in 2 barns (70 and 72) at end of 2021 and included data collection of 32 cows with the measurment frequency set to 10Hz and
the accelerometers activated. The second part of the intallation in September 2022 entailed 4 additional barns and the waiting area (with 5 antennas).
As the UWB system monitors positions of the tags, the tag id needs to be coupled with the cow id in order to render the information useful.
Battery life of the tag depends on measurement frequency and activation of additional sensors (some have acceleration, gyroscope and magnetometer mounted) and
therefore varies between approx. 14 and 90 days.
The data are automatically written to a sewio-database that can be accessed via the [IP address](http://10.82.16.215/login) and login information. This
gives access to the database and real-time visualisation tools,.
[sensmap visualisation](/documentation/sensmap_visualisation.png)
### Tools
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment