How to use the gokart.target.LargeDataFrameProcessor function in gokart

To help you get started, we’ve selected a few gokart examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github m3dev / gokart / test / test_large_data_fram_processor.py View on Github external
def test_save_and_load_empty(self):
        file_path = os.path.join(_get_temporary_directory(), 'test_with_empty.zip')
        df = pd.DataFrame()
        processor = LargeDataFrameProcessor(max_byte=int(1e+6))
        processor.save(df, file_path)
        loaded = processor.load(file_path)

        pd.testing.assert_frame_equal(loaded, df, check_like=True)
github m3dev / gokart / test / test_large_data_fram_processor.py View on Github external
def test_save_and_load(self):
        file_path = os.path.join(_get_temporary_directory(), 'test.zip')
        df = pd.DataFrame(dict(data=np.random.uniform(0, 1, size=int(1e+6))))
        processor = LargeDataFrameProcessor(max_byte=int(1e+6))
        processor.save(df, file_path)
        loaded = processor.load(file_path)

        pd.testing.assert_frame_equal(loaded, df, check_like=True)
github m3dev / gokart / gokart / task.py View on Github external
def make_large_data_frame_target(self, relative_file_path: str, use_unique_id: bool = True, max_byte=int(2**26)) -> TargetOnKart:
        file_path = os.path.join(self.workspace_directory, relative_file_path)
        unique_id = self.make_unique_id() if use_unique_id else None
        return gokart.target.make_model_target(file_path=file_path,
                                               temporary_directory=self.local_temporary_directory,
                                               unique_id=unique_id,
                                               save_function=gokart.target.LargeDataFrameProcessor(max_byte=max_byte).save,
                                               load_function=gokart.target.LargeDataFrameProcessor.load)
github m3dev / gokart / gokart / task.py View on Github external
def make_large_data_frame_target(self, relative_file_path: str, use_unique_id: bool = True, max_byte=int(2**26)) -> TargetOnKart:
        file_path = os.path.join(self.workspace_directory, relative_file_path)
        unique_id = self.make_unique_id() if use_unique_id else None
        return gokart.target.make_model_target(file_path=file_path,
                                               temporary_directory=self.local_temporary_directory,
                                               unique_id=unique_id,
                                               save_function=gokart.target.LargeDataFrameProcessor(max_byte=max_byte).save,
                                               load_function=gokart.target.LargeDataFrameProcessor.load)