@@ -270,19 +270,24 @@ def _run_multi_process(self, data_frame: DataFrame, wait=True, timeout=None):
270
270
timeout (Union[int, float]): ``concurrent.futures.TimeoutError`` will be raised
271
271
if timeout is reached.
272
272
"""
273
+ # pylint: disable=I1101
273
274
batch_size = math .ceil (data_frame .shape [0 ] / self .max_processes )
275
+ # pylint: enable=I1101
274
276
275
277
args = []
276
278
for i in range (self .max_processes ):
277
279
start_index = min (i * batch_size , data_frame .shape [0 ])
278
280
end_index = min (i * batch_size + batch_size , data_frame .shape [0 ])
279
- args += [(
280
- self .max_workers ,
281
- self .feature_group_name ,
282
- self .sagemaker_fs_runtime_client_config ,
283
- data_frame [start_index :end_index ],
284
- start_index ,
285
- timeout )]
281
+ args += [
282
+ (
283
+ self .max_workers ,
284
+ self .feature_group_name ,
285
+ self .sagemaker_fs_runtime_client_config ,
286
+ data_frame [start_index :end_index ],
287
+ start_index ,
288
+ timeout ,
289
+ )
290
+ ]
286
291
287
292
def init_worker ():
288
293
# ignore keyboard interrupts in child processes.
@@ -298,10 +303,14 @@ def init_worker():
298
303
self .wait (timeout = timeout )
299
304
300
305
def _run_multi_threaded (
301
- self ,
302
- max_workers : int , feature_group_name : str ,
303
- sagemaker_fs_runtime_client_config : Config ,
304
- data_frame : DataFrame , row_offset = 0 , timeout = None ) -> List [int ]:
306
+ self ,
307
+ max_workers : int ,
308
+ feature_group_name : str ,
309
+ sagemaker_fs_runtime_client_config : Config ,
310
+ data_frame : DataFrame ,
311
+ row_offset = 0 ,
312
+ timeout = None ,
313
+ ) -> List [int ]:
305
314
"""Start the ingestion process.
306
315
307
316
Args:
@@ -316,7 +325,9 @@ def _run_multi_threaded(
316
325
List of row indices that failed to be ingested.
317
326
"""
318
327
executor = ThreadPoolExecutor (max_workers = max_workers )
328
+ # pylint: disable=I1101
319
329
batch_size = math .ceil (data_frame .shape [0 ] / max_workers )
330
+ # pylint: enable=I1101
320
331
321
332
futures = {}
322
333
for i in range (max_workers ):
0 commit comments