Merge pull request #518 from AllenInstitute/fix/linting

This addresses issue #513 and makes PR #514 unnecessary. For future reference, here's the explanation for the #513 fix: 

This PR addresses issue #513 by changing all of the calls to pandas.read_hdf() to no longer pass 'format' as a kwarg. We used to do this, and the passed kwarg was ignored by HDFStore.open because it wasn't a defined keyword arg for that method, but no alarms were raised.

See: pandas-dev/pandas#13291

In newer versions of pandas there is an exception raised if you try to pass 'format' to the read_hdf() method. Removing this kwarg:

- Won't change the output of read_hdf, as the kwarg was being ignored before, and
- Allows me to run Marina's old notebooks with VBA analysis code and pandas==0.24.2

Author: nickponvert

visual_behavior/ophys/dataset/cell_matching_dataset.py

@@ -85,10 +85,10 @@ def get_analysis_folder_name(self):
             lims_data = self.lims_data
             date = str(lims_data.experiment_date)[:10].split('-')
             analysis_folder_name = str(lims_data.lims_id) + '_' + \
-                                   str(lims_data.mouse_id) + '_' + date[0][2:] + date[1] + date[2] + '_' + \
-                                   lims_data.structure + '_' + str(lims_data.depth) + '_' + \
-                                   lims_data.specimen_driver_line.split('-')[0] + '_' + lims_data.rig[3:5] + \
-                                   lims_data.rig[6] + '_' + lims_data.session_type
+                str(lims_data.mouse_id) + '_' + date[0][2:] + date[1] + date[2] + '_' + \
+                lims_data.structure + '_' + str(lims_data.depth) + '_' + \
+                lims_data.specimen_driver_line.split('-')[0] + '_' + lims_data.rig[3:5] + \
+                lims_data.rig[6] + '_' + lims_data.session_type
             self.analysis_folder_name = analysis_folder_name
         return self.analysis_folder_name
 

visual_behavior/ophys/dataset/visual_behavior_ophys_dataset.py

@@ -96,13 +96,13 @@ def get_analysis_dir(self):
     analysis_dir = LazyLoadable('_analysis_dir', get_analysis_dir)
 
     def get_metadata(self):
-        self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df', format='fixed')
+        self._metadata = pd.read_hdf(os.path.join(self.analysis_dir, 'metadata.h5'), key='df')
         return self._metadata
 
     metadata = LazyLoadable('_metadata', get_metadata)
 
     def get_timestamps(self):
-        self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df', format='fixed')
+        self._timestamps = pd.read_hdf(os.path.join(self.analysis_dir, 'timestamps.h5'), key='df')
         return self._timestamps
 
     timestamps = LazyLoadable('_timestamps', get_timestamps)
@@ -122,7 +122,7 @@ def get_timestamps_ophys(self):
     def get_stimulus_table(self):
         self._stimulus_table = pd.read_hdf(
             os.path.join(self.analysis_dir, 'stimulus_table.h5'),
-            key='df', format='fixed'
+            key='df'
         )
         self._stimulus_table = self._stimulus_table.reset_index()
         # self._stimulus_table = self._stimulus_table.drop(
@@ -146,42 +146,42 @@ def get_stimulus_template(self):
     def get_stimulus_metadata(self):
         self._stimulus_metadata = pd.read_hdf(
             os.path.join(self.analysis_dir, 'stimulus_metadata.h5'),
-            key='df', format='fixed'
+            key='df'
         )
         self._stimulus_metadata = self._stimulus_metadata.drop(columns='image_category')
         return self._stimulus_metadata
 
     stimulus_metadata = LazyLoadable('_stimulus_metadata', get_stimulus_metadata)
 
     def get_running_speed(self):
-        self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df', format='fixed')
+        self._running_speed = pd.read_hdf(os.path.join(self.analysis_dir, 'running_speed.h5'), key='df')
         return self._running_speed
 
     running_speed = LazyLoadable('_running_speed', get_running_speed)
 
     def get_licks(self):
-        self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df', format='fixed')
+        self._licks = pd.read_hdf(os.path.join(self.analysis_dir, 'licks.h5'), key='df')
         return self._licks
 
     licks = LazyLoadable('_licks', get_licks)
 
     def get_rewards(self):
-        self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df', format='fixed')
+        self._rewards = pd.read_hdf(os.path.join(self.analysis_dir, 'rewards.h5'), key='df')
         return self._rewards
 
     rewards = LazyLoadable('_rewards', get_rewards)
 
     def get_task_parameters(self):
         self._task_parameters = pd.read_hdf(
             os.path.join(self.analysis_dir, 'task_parameters.h5'),
-            key='df', format='fixed'
+            key='df'
         )
         return self._task_parameters
 
     task_parameters = LazyLoadable('_task_parameters', get_task_parameters)
 
     def get_all_trials(self):
-        self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df', format='fixed')
+        self._all_trials = pd.read_hdf(os.path.join(self.analysis_dir, 'trials.h5'), key='df')
         return self._all_trials
 
     all_trials = LazyLoadable('_all_trials', get_all_trials)
@@ -255,13 +255,12 @@ def get_events(self):
     events = LazyLoadable('_events', get_events)
 
     def get_roi_metrics(self):
-        self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df', format='fixed')
+        self._roi_metrics = pd.read_hdf(os.path.join(self.analysis_dir, 'roi_metrics.h5'), key='df')
         return self._roi_metrics
 
     roi_metrics = LazyLoadable('_roi_metrics', get_roi_metrics)
 
     def get_roi_mask_dict(self):
-        cell_specimen_ids = self.get_cell_specimen_ids()
         f = h5py.File(os.path.join(self.analysis_dir, 'roi_masks.h5'), 'r')
         roi_mask_dict = {}
         for key in f.keys():
@@ -300,15 +299,15 @@ def get_average_image(self):
     def get_motion_correction(self):
         self._motion_correction = pd.read_hdf(
             os.path.join(self.analysis_dir, 'motion_correction.h5'),
-            key='df', format='fixed'
+            key='df'
         )
         return self._motion_correction
 
     motion_correction = LazyLoadable('_motion_correction', get_motion_correction)
 
     def get_cell_specimen_ids(self):
         roi_metrics = self.roi_metrics
-        self._cell_specimen_ids = np.asarray([roi_metrics[roi_metrics.roi_id==roi_id].id.values[0]
+        self._cell_specimen_ids = np.asarray([roi_metrics[roi_metrics.roi_id == roi_id].id.values[0]
                                               for roi_id in np.sort(self.roi_metrics.roi_id.values)])
         return self._cell_specimen_ids
 
@@ -322,12 +321,12 @@ def get_cell_indices(self):
 
     def get_cell_specimen_id_for_cell_index(self, cell_index):
         roi_metrics = self.roi_metrics
-        cell_specimen_id = roi_metrics[roi_metrics.cell_index==cell_index].id.values[0]
+        cell_specimen_id = roi_metrics[roi_metrics.cell_index == cell_index].id.values[0]
         return cell_specimen_id
 
     def get_cell_index_for_cell_specimen_id(self, cell_specimen_id):
         roi_metrics = self.roi_metrics
-        cell_index = roi_metrics[roi_metrics.id==cell_specimen_id].cell_index.values[0]
+        cell_index = roi_metrics[roi_metrics.id == cell_specimen_id].cell_index.values[0]
         return cell_index
 
     @classmethod

visual_behavior/ophys/response_analysis/response_analysis.py

@@ -42,10 +42,10 @@ def __init__(self, dataset, overwrite_analysis_files=False, use_events=False):
         self.trial_window = [-4, 8]  # time, in seconds, around change time to extract portion of cell trace
         self.flash_window = [-0.5,
                              0.75]  # time, in seconds, around stimulus flash onset time to extract portion of cell trace
-        self.omitted_flash_window = [-3,3]
+        self.omitted_flash_window = [-3, 3]
         self.response_window_duration = 0.5  # window, in seconds, over which to take the mean for a given trial or flash
         self.response_window = [np.abs(self.trial_window[0]), np.abs(self.trial_window[
-                                0]) + self.response_window_duration]  # time, in seconds, around change time to take the mean response
+            0]) + self.response_window_duration]  # time, in seconds, around change time to take the mean response
         self.baseline_window = np.asarray(
             self.response_window) - self.response_window_duration  # time, in seconds, relative to change time to take baseline mean response
         self.stimulus_duration = 0.25  # self.dataset.task_parameters['stimulus_duration'].values[0]
@@ -98,14 +98,16 @@ def generate_trial_response_df(self):
                 mean_running_speed = ut.get_mean_in_window(running_speed_trace, self.response_window,
                                                            self.stimulus_frame_rate)
                 df_list.append(
-                    [trial, int(cell_index), int(cell_specimen_id), trace, timestamps, mean_response, baseline_response, n_events,
-                     p_value, sd_over_baseline, mean_running_speed, cell_matching_has_been_run, self.dataset.experiment_id])
-                     #running_speed_trace, running_speed_timestamps,
-
+                    [trial, int(cell_index), int(cell_specimen_id), trace, timestamps, mean_response, baseline_response,
+                     n_events,
+                     p_value, sd_over_baseline, mean_running_speed, cell_matching_has_been_run,
+                     self.dataset.experiment_id])
+                # running_speed_trace, running_speed_timestamps,
 
         columns = ['trial', 'cell', 'cell_specimen_id', 'trace', 'timestamps', 'mean_response', 'baseline_response',
-                   'n_events', 'p_value', 'sd_over_baseline', 'mean_running_speed', 'cell_matching_has_been_run', 'experiment_id']
-                    #'running_speed_trace', 'running_speed_timestamps',
+                   'n_events', 'p_value', 'sd_over_baseline', 'mean_running_speed', 'cell_matching_has_been_run',
+                   'experiment_id']
+        # 'running_speed_trace', 'running_speed_timestamps',
         trial_response_df = pd.DataFrame(df_list, columns=columns)
         trial_metadata = self.dataset.trials
         trial_metadata = trial_metadata.rename(columns={'response': 'behavioral_response'})
@@ -123,7 +125,6 @@ def save_trial_response_df(self, trial_response_df):
     def get_trial_response_df(self):
         if self.overwrite_analysis_files:
             print('overwriting analysis files')
-            import h5py
             file_path = self.get_trial_response_df_path()
             if os.path.exists(file_path):
                 os.remove(file_path)
@@ -132,7 +133,7 @@ def get_trial_response_df(self):
         else:
             if os.path.exists(self.get_trial_response_df_path()):
                 print('loading trial response dataframe')
-                self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df', format='fixed')
+                self.trial_response_df = pd.read_hdf(self.get_trial_response_df_path(), key='df')
                 tdf = self.trial_response_df
                 tdf.cell = [int(cell) for cell in tdf.cell.values]
                 tdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in tdf.cell_specimen_id.values]
@@ -195,15 +196,19 @@ def generate_flash_response_df(self):
                                                            self.stimulus_frame_rate)
                 reward_rate = flash_data.reward_rate.values[0]
 
-                row.append([int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
-                            trace, timestamps, mean_response, baseline_response, n_events, p_value_baseline, sd_over_baseline,
-                            reward_rate, mean_running_speed, cell_matching_has_been_run, int(self.dataset.experiment_id)])
+                row.append(
+                    [int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
+                     trace, timestamps, mean_response, baseline_response, n_events, p_value_baseline, sd_over_baseline,
+                     reward_rate, mean_running_speed, cell_matching_has_been_run, int(self.dataset.experiment_id)])
 
         flash_response_df = pd.DataFrame(data=row,
                                          columns=['cell', 'cell_specimen_id', 'flash_number', 'omitted', 'start_time',
-                                                  'image_name', 'image_category', 'trace', 'timestamps', 'mean_response',
-                                                  'baseline_response', 'n_events', 'p_value_baseline', 'sd_over_baseline',
-                                                  'reward_rate', 'mean_running_speed', 'cell_matching_has_been_run', 'experiment_id'])
+                                                  'image_name', 'image_category', 'trace', 'timestamps',
+                                                  'mean_response',
+                                                  'baseline_response', 'n_events', 'p_value_baseline',
+                                                  'sd_over_baseline',
+                                                  'reward_rate', 'mean_running_speed', 'cell_matching_has_been_run',
+                                                  'experiment_id'])
         flash_response_df = ut.annotate_flash_response_df_with_pref_stim(flash_response_df)
         flash_response_df = ut.add_repeat_number_to_flash_response_df(flash_response_df, stimulus_table)
         flash_response_df = ut.add_image_block_to_flash_response_df(flash_response_df, stimulus_table)
@@ -238,7 +243,6 @@ def save_flash_response_df(self, flash_response_df):
     def get_flash_response_df(self):
         if self.overwrite_analysis_files:
             # delete old file or else it will keep growing in size
-            import h5py
             file_path = self.get_flash_response_df_path()
             if os.path.exists(file_path):
                 os.remove(file_path)
@@ -247,7 +251,7 @@ def get_flash_response_df(self):
         else:
             if os.path.exists(self.get_flash_response_df_path()):
                 print('loading flash response dataframe')
-                self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df', format='fixed')
+                self.flash_response_df = pd.read_hdf(self.get_flash_response_df_path(), key='df')
                 fdf = self.flash_response_df
                 fdf.cell = [int(cell) for cell in fdf.cell.values]
                 fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values]
@@ -268,7 +272,7 @@ def get_omitted_flash_response_df_path(self):
     def generate_omitted_flash_response_df(self):
         print('generating omitted flash response df')
         stimulus_table = ut.annotate_flashes_with_reward_rate(self.dataset)
-        stimulus_table = stimulus_table[stimulus_table.omitted==True]
+        stimulus_table = stimulus_table[stimulus_table.omitted == True]
         row = []
         for cell in self.dataset.cell_indices:
             cell = int(cell)
@@ -304,13 +308,15 @@ def generate_omitted_flash_response_df(self):
                 n_events = ut.get_n_nonzero_in_window(trace, response_window, self.ophys_frame_rate)
                 reward_rate = flash_data.reward_rate.values[0]
 
-                row.append([int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
-                            trace, timestamps, mean_response, baseline_response, n_events, p_value, sd_over_baseline,
-                            reward_rate, cell_matching_has_been_run, int(self.dataset.experiment_id)])
+                row.append(
+                    [int(cell), int(cell_specimen_id), int(flash), omitted, flash_time, image_name, image_category,
+                     trace, timestamps, mean_response, baseline_response, n_events, p_value, sd_over_baseline,
+                     reward_rate, cell_matching_has_been_run, int(self.dataset.experiment_id)])
 
         flash_response_df = pd.DataFrame(data=row,
                                          columns=['cell', 'cell_specimen_id', 'flash_number', 'omitted', 'start_time',
-                                                  'image_name', 'image_category', 'trace', 'timestamps', 'mean_response',
+                                                  'image_name', 'image_category', 'trace', 'timestamps',
+                                                  'mean_response',
                                                   'baseline_response', 'n_events', 'p_value', 'sd_over_baseline',
                                                   'reward_rate', 'cell_matching_has_been_run', 'experiment_id'])
         # flash_response_df = ut.annotate_flash_response_df_with_pref_stim(flash_response_df)
@@ -325,7 +331,6 @@ def save_omitted_flash_response_df(self, omitted_flash_response_df):
     def get_omitted_flash_response_df(self):
         if self.overwrite_analysis_files:
             # delete old file or else it will keep growing in size
-            import h5py
             file_path = self.get_omitted_flash_response_df_path()
             if os.path.exists(file_path):
                 os.remove(file_path)
@@ -334,7 +339,7 @@ def get_omitted_flash_response_df(self):
         else:
             if os.path.exists(self.get_omitted_flash_response_df_path()):
                 print('loading omitted flash response dataframe')
-                self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df', format='fixed')
+                self.omitted_flash_response_df = pd.read_hdf(self.get_omitted_flash_response_df_path(), key='df')
                 fdf = self.omitted_flash_response_df
                 fdf.cell = [int(cell) for cell in fdf.cell.values]
                 fdf.cell_specimen_id = [int(cell_specimen_id) for cell_specimen_id in fdf.cell_specimen_id.values]
@@ -344,7 +349,6 @@ def get_omitted_flash_response_df(self):
                 self.save_omitted_flash_response_df(self.omitted_flash_response_df)
         return self.omitted_flash_response_df
 
-
     def compute_pairwise_correlations(self):
         fdf = self.flash_response_df.copy()
         if 'omitted' in fdf.keys():