Do you have a dataset that needs to be gap-filled?
In this notebook we repeat the analysis for user supplied data.
Download and prepare the dataset¶
from erddapy import ERDDAP
import os
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import panel as pn
import holoviews as hv
from holoviews import opts
hv.extension('bokeh')
pn.extension()df = pd.read_csv('dataset.csv', parse_dates=True, index_col=0)dfExplore the data¶
def plot_all_sites(df, cmap='Viridis'):
image_data = df.astype('float32').T.values
x_labels = df.index.strftime('%Y-%m-%d') # dates → x-axis
y_labels = list(df.columns) # station-depths → y-axis
x_coords = np.arange(len(x_labels))
y_coords = np.arange(len(y_labels))
heatmap = hv.Image((x_coords, y_coords, image_data)).opts(
xaxis='bottom',
xlabel='Date',
ylabel='Station @ Depth',
xticks=list(zip(x_coords[::30], x_labels[::30])), # every 30th date
yticks=list(zip(y_coords, y_labels)),
xrotation=45,
cmap=cmap,
colorbar=True,
width=1000,
height=800,
tools=['hover']
)
return heatmap
plot_all_sites(df)Visualize the series data¶
# Create a dropdown selector
site_selector = pn.widgets.Select(name='Site', options=list(df.columns))
def highlight_nan_regions(label):
series = df[label]
# Identify NaN regions
is_nan = series.isna()
nan_ranges = []
current_start = None
for date, missing in is_nan.items():
if missing and current_start is None:
current_start = date
elif not missing and current_start is not None:
nan_ranges.append((current_start, date))
current_start = None
if current_start is not None:
nan_ranges.append((current_start, series.index[-1]))
# Create shaded regions
spans = [
hv.VSpan(start, end).opts(color='red', alpha=0.2)
for start, end in nan_ranges
]
curve = hv.Curve(series, label=label).opts(
width=900, height=250, tools=['hover', 'box_zoom', 'pan', 'wheel_zoom'],
show_grid=True, title=label
)
return curve * hv.Overlay(spans)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[1], line 2
1 # Create a dropdown selector
----> 2 site_selector = pn.widgets.Select(name='Site', options=list(df.columns))
4 def highlight_nan_regions(label):
6 series = df[label]
NameError: name 'pn' is not defined
interactive_plot = hv.DynamicMap(pn.bind(highlight_nan_regions, site_selector))
pn.Column(site_selector, interactive_plot, 'Hightlights regions are gaps that need to imputed.')Impute the gaps¶
We have determined that the MissForestappears to work reasonably well when imputing artificially large gaps.
We use it to gap fill the missing data in this dataset.
from imputeMF import imputeMFdf_imputed = pd.DataFrame(imputeMF(df.values, 10, print_stats=True), columns=df.columns, index=df.index)Save the imputed dataset
df_imputed.to_csv('dataset_imputed.csv')---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[2], line 1
----> 1 df_imputed.to_csv('dataset_imputed.csv')
NameError: name 'df_imputed' is not defineddf_imputeddef highlight_imputed_regions(label):
series = df[label]
series_imputed = df_imputed[label]
# Identify NaN regions
is_nan = series.isna()
nan_ranges = []
current_start = None
for date, missing in is_nan.items():
if missing and current_start is None:
current_start = date
elif not missing and current_start is not None:
nan_ranges.append((current_start, date))
current_start = None
if current_start is not None:
nan_ranges.append((current_start, series.index[-1]))
# Create shaded regions
spans = [
hv.VSpan(start, end).opts(color='red', alpha=0.2)
for start, end in nan_ranges
]
curve = hv.Curve(series_imputed, label=label).opts(
width=900, height=250, tools=['hover', 'box_zoom', 'pan', 'wheel_zoom'],
show_grid=True, title=label
)
return curve * hv.Overlay(spans)
interactive_plot = hv.DynamicMap(pn.bind(highlight_imputed_regions, site_selector))
pn.Column(site_selector, interactive_plot)Highlighted regions show where the gaps have been imputed.
Notice the imputation algorithm gap fills in time intervals where there is very limited information from any other site. Care should be taken in interpretation of interpolated data.
plot_all_sites(df_imputed)