Narrow River kayak trip data processing notebook¶
This is a Jupyter notebook. They are neat, you can read about them here:http://jupyter.org/.
They are made up of cells containing:
- inline markdown like this (which can contain code snippets or LaTex / MathJax)
- raw code to be run interactively
- example code not to be run
- raw html/CSS/JS
- any output such as figures, tables, console, etc.
- there are kernels / engines for over 40 languages
- converting to script, pdf, html, or slideshow (with reveal.js)
Or to quote from the source "Open source, interactive data science and scientific computing across over 40 programming languages."
This one will outline some routines for processing temperature data logged during the inaugural PODS kayak trip.¶
In [1]:
# traditional python imports at the beginning
import numpy as np # matrix library
import matplotlib.pyplot as plt # plotting library
import pandas as pd # dataframe / spreadsheet library with some nice extras for timeseries etc.
# nbagg for interactive plotting within the notebook
# uline for static/inline plots
%matplotlib inline
plt.rcParams['figure.figsize'] = (16, 9)
plt.rcParams['axes.formatter.useoffset'] = False
#plt.rcParams['axes.formatter'] = False
In [2]:
#hideme
for i in plt.rcParams.keys():
print i
In [3]:
# function definitions should go here so they are loaded from the beginning...
def readTemp(fname):
"""
convenience funtion to read in hobo csv's, strip of unwanted data,
and set index to DateTime parsed from file and trim start to be around
launch time
returns pandas DataFrame object of single temp record indexed by time
that can then be merged etc.
"""
temp = pd.read_csv(fname, header=1, parse_dates=[1]) # read the file in
temp.drop(temp.columns[[0]], axis=1, inplace=True) # drop the first index column it is redundant
temp.set_index('Date Time, GMT-04:00', inplace=True) # set index to be time not integer - can use iloc and loc
# trim to time we were in water, start at 2:30 local to 3:30 local
temp = temp['2016-09-28 14:45:00':'2016-09-28 15:40:00']
return temp
In [4]:
# list out the files in the directory
!ls
In [5]:
# lets try importing one of the temp logs
temp1 = readTemp('NR_2016-09-28_T1.csv')
temp2 = readTemp('NR_2016-09-28_T2.csv')
temp2.head()
Out[5]:
In [6]:
# joining / merging datasets
#mytemps = pd.merge(temp1, temp2, on='Date Time, GMT-04:00', how='outer')
#mytemps = pd.merge(temp1, temp2, how='outer')
mytemps = temp1.join(temp2, how='outer') # outer join keeps everything and fills in blanks with NANs
mytemps.head() # look at the top
Out[6]:
In [7]:
# renaming column headers
'''
mytemps.rename(columns={'Temp, °C (LGR S/N: 11001004, SEN S/N: 11001004)': 'T1',
'Temp, °C (LGR S/N: 11000995, SEN S/N: 11000995)': 'T2'},
inplace=True)
'''
mytemps.columns = ['T1','T2']
In [8]:
mytemps['Tavg'] = (mytemps['T1'] + mytemps['T2']) / 2.0 # add an average column
mytemps['Tanom'] = mytemps['Tavg'] - mytemps['Tavg'].mean()
In [9]:
mytemps.describe() # look at some statistics
Out[9]:
In [10]:
mytemps.info() # more info on object / data
In [11]:
# plot up timeseries
#plt.figure()
mytemps.plot(subplots=True)
plt.legend(loc='best')
plt.ylabel(u'Temp, °C')
plt.title('Tucker towed HOBO time series');
In [12]:
plt.figure()
mytemps['T1'].plot()
mytemps['T2'].plot()
mytemps['Tavg'].plot()
mean = mytemps['Tavg'].mean()
meanlabel = u'Mean = {:4.1f}°C'.format(mean)
plt.axhline(y=mean, xmin=0, xmax=1, hold=None, label=meanlabel, color='teal', lw=4)
plt.legend()
Out[12]:
Adding GPS and georeferencing temperatures¶
In [13]:
# read in gps data
gpsTrack = pd.read_csv('kayak-track-only.csv', header=0, parse_dates=[0]) # read and parse headers/dates
gpsTrack.set_index('time GMT', inplace=True) # set index to be time
gpsTrack.index = gpsTrack.index - pd.offsets.Hour(4) # subtract 4 hrs to put in local time
gpsTrack.index.rename('local time', inplace=True) # rename the index to reflect local time change
geotemps = gpsTrack.join(mytemps, how='inner') # inner join = keep common indices to both datasets
geotemps.head() # show the top
Out[13]:
In [14]:
geotemps.tail() # look at the end
Out[14]:
In [15]:
geotemps.describe() # statistics
Out[15]:
In [16]:
# plotting timeseries
#plt.figure()
geotemps.plot(subplots=True); # put all on separate subplots
In [23]:
# more plotting
plt.figure()
#fig.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
cm = plt.cm.get_cmap('winter')
x = geotemps['longitude']
y = geotemps['latitude']
c = geotemps['Tavg'] - geotemps['Tavg'].mean()
# color normalization
#c /= c.max()
#colors = cm.seismic(np.linspace(0,1,len(c)))
'''
# plot point by point - probably not best way
for i in range(len(x)):
plt.scatter(x[i],y[i], color=colors[i], vmin=18, vmax=20, s=55, cmap=cm)
'''
# plot whole list at once
sc = plt.scatter(x, y, c=c, s=55, cmap=cm, alpha=0.8, label='T')
c = plt.colorbar(sc, orientation='horizontal')
c.set_label('T')
plt.xlim((x.min(), x.max()))
plt.ylim((y.min(),y.max()))
plt.xlabel('Longitude')
plt.ylabel('Latitude')
#plt.legend()
titlestr = u'Temperature anomaly [°C] at mouth of Narrow River'
plt.title(titlestr);
#plt.show()
In [18]:
# more plotting
plt.figure()
#fig.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
cm = plt.cm.get_cmap('rainbow')
x = geotemps['longitude']
y = geotemps['latitude']
c = geotemps.index
# color normalization
#c /= c.max()
#colors = cm.seismic(np.linspace(0,1,len(c)))
'''
# plot point by point - probably not best way
for i in range(len(x)):
plt.scatter(x[i],y[i], color=colors[i], vmin=18, vmax=20, s=55, cmap=cm)
'''
# plot whole list at once
sc = plt.scatter(x, y, c=c, s=70, cmap=cm, alpha=0.8, label='time')
plt.plot(x,y,'k')
#c = plt.colorbar(sc, orientation='horizontal')
#c.set_label('T')
plt.xlim((x.min(), x.max()))
plt.ylim((y.min(),y.max()))
plt.xlabel('Longitude')
plt.ylabel('Latitude')
plt.legend()
titlestr = 'Track time log, lightens toward end'
plt.title(titlestr);
#plt.show()
plotting on a map... maybe¶
In [19]:
# basemap plotting
from mpl_toolkits.basemap import Basemap
In [20]:
plt.figure()
m = Basemap()
m.drawcoastlines()
plt.show()
In [21]:
plt.figure(figsize=(9,16))
minlon = geotemps['longitude'].min()
minlat = geotemps['latitude'].min()
maxlon = geotemps['longitude'].max()
maxlat = geotemps['latitude'].max()
meanlon = geotemps['longitude'].mean()
meanlat = geotemps['latitude'].mean()
m = Basemap(projection='merc',
lat_0=meanlat,
lon_0=meanlon,
resolution='h',
area_thresh=0.1,
llcrnrlon=-71.5,
llcrnrlat=41.4,
urcrnrlon=-71.4,
urcrnrlat=41.5,
)
#m.drawmapboundary(fill_color='blue')
#m.fillcontinents(color='coral', lake_color='aqua')
#m.bluemarble()
#m.drawcoastlines()
#m.drawrivers()
m.readshapefile('./rhode_island_water/rhode_island_water', 'water')
m.readshapefile('./rhode_island_coastline/rhode_island_coastline', 'coast')
#wx, wy = zip(*m.water)
#m.plot(wx,wy,'k')
# scatter our data on the map colored by temp anomoly
x1, y1 = m(geotemps['longitude'].values, geotemps['latitude'].values) # cast into map coordinates [m]
c = geotemps['Tanom']
m.scatter(x1, y1, c=c, s=30, cmap='seismic', zorder=5)
plt.show()
In [24]:
plt.figure(figsize=(9,16))
minlon = geotemps['longitude'].min()
minlat = geotemps['latitude'].min()
maxlon = geotemps['longitude'].max()
maxlat = geotemps['latitude'].max()
meanlon = geotemps['longitude'].mean()
meanlat = geotemps['latitude'].mean()
m = Basemap(projection='merc',
lat_0=meanlat,
lon_0=meanlon,
resolution='h',
area_thresh=0.001,
llcrnrlon=-71.452,
llcrnrlat=41.44,
urcrnrlon=-71.442,
urcrnrlat=41.455,
)
#m.drawmapboundary(fill_color='blue')
#m.fillcontinents(color='coral', lake_color='aqua')
#m.bluemarble()
#m.drawcoastlines()
#m.drawrivers()
m.readshapefile('./rhode_island_water/rhode_island_water', 'water')
m.readshapefile('./rhode_island_coastline/rhode_island_coastline', 'coast')
#wx, wy = zip(*m.water)
#m.plot(wx,wy,'k')
# scatter our data on the map colored by temp anomoly
x1, y1 = m(geotemps['longitude'].values, geotemps['latitude'].values) # cast into map coordinates [m]
c = geotemps['Tanom']
m.scatter(x1, y1, c=c, s=70, cmap='winter', zorder=5)
plt.show()
In [38]:
t9 = readTemp('./NR_2016-09-28_T9.csv')
t8 = readTemp('./NR_2016-09-28_T8.csv')
t10 = readTemp('./NR_2016-09-28_T10.csv')
t6 = readTemp('./NR_2016-09-28_T6.csv')
t12 = readTemp('./NR_2016-09-28_T12.csv')
In [40]:
%matplotlib nbagg
In [42]:
plt.figure(figsize=(16,9))
plt.hold(True)
plt.plot(t9['Temp, \xc2\xb0C (LGR S/N: 11001009, SEN S/N: 11001009)'].values, label='t9')
plt.plot(t8['Temp, \xc2\xb0C (LGR S/N: 11000997, SEN S/N: 11000997)'].values, label='t8')
plt.plot(t10['Temp, \xc2\xb0C (LGR S/N: 11001008, SEN S/N: 11001008)'].values, label='t10')
plt.plot(t6['Temp, \xc2\xb0C (LGR S/N: 11000996, SEN S/N: 11000996)'].values, label='t6')
plt.plot(t12['Temp, \xc2\xb0C (LGR S/N: 11001006, SEN S/N: 11001006)'].values, label='t12')
plt.plot(mytemps['T1'].values, label='t1')
plt.plot(mytemps['T2'].values, label='t2')
plt.legend()
Out[42]: