How to use the mindsdb.config.CONFIG.MINDSDB_STORAGE_PATH function in MindsDB

To help you get started, we’ve selected a few MindsDB examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mindsdb / mindsdb / mindsdb / libs / controllers / predictor.py View on Github external
storage_ok = True  # default state

        # if it does not exist try to create it
        if not os.path.exists(CONFIG.MINDSDB_STORAGE_PATH):
            try:
                self.log.info('{folder} does not exist, creating it now'.format(folder=CONFIG.MINDSDB_STORAGE_PATH))
                path = Path(CONFIG.MINDSDB_STORAGE_PATH)
                path.mkdir(exist_ok=True, parents=True)
            except:
                self.log.info(traceback.format_exc())
                storage_ok = False
                self.log.error('MindsDB storage foldler: {folder} does not exist and could not be created'.format(
                    folder=CONFIG.MINDSDB_STORAGE_PATH))

        # If storage path is not writable, raise an exception as this can no longer be
        if not os.access(CONFIG.MINDSDB_STORAGE_PATH, os.W_OK) or storage_ok == False:
            error_message = '''Cannot write into storage path, please either set the config variable mindsdb.config.set('MINDSDB_STORAGE_PATH',
github mindsdb / mindsdb / mindsdb / libs / controllers / predictor.py View on Github external
    def __init__(self, name, root_folder=CONFIG.MINDSDB_STORAGE_PATH, log_level=CONFIG.DEFAULT_LOG_LEVEL, log_server=CONFIG.MINDSDB_SERVER_URL):
        """
        This controller defines the API to a MindsDB 'mind', a mind is an object that can learn and predict from data

        :param name: the namespace you want to identify this mind instance with
        :param root_folder: the folder where you want to store this mind or load from
        :param log_level: the desired log level
        :param log_server: the url for a server that can accept log streams

        """

        # initialize variables
        self.name = name
        self.root_folder = root_folder
        self.uuid = str(uuid.uuid1())
        self.predict_worker = None
github mindsdb / mindsdb / mindsdb / libs / controllers / predictor.py View on Github external
    def __init__(self, name, root_folder=CONFIG.MINDSDB_STORAGE_PATH, log_level=CONFIG.DEFAULT_LOG_LEVEL):
        """
        This controller defines the API to a MindsDB 'mind', a mind is an object that can learn and predict from data

        :param name: the namespace you want to identify this mind instance with
        :param root_folder: the folder where you want to store this mind or load from
        :param log_level: the desired log level

        """

        # initialize variables
        self.name = name
        self.root_folder = root_folder
        self.uuid = str(uuid.uuid1())
        # initialize log
        self.log = MindsdbLogger(log_level=log_level, uuid=self.uuid)
github mindsdb / mindsdb / mindsdb / libs / helpers / general_helpers.py View on Github external
def check_for_updates():
    """
    Check for updates of mindsdb
    it will ask the mindsdb server if there are new versions, if there are it will log a message

    :return: None
    """

    # tmp files
    uuid_file = CONFIG.MINDSDB_STORAGE_PATH + '/../uuid.mdb_base'
    mdb_file = CONFIG.MINDSDB_STORAGE_PATH + '/start.mdb_base'

    uuid_file_path = Path(uuid_file)
    if uuid_file_path.is_file():
        uuid_str = open(uuid_file).read()
    else:
        uuid_str = str(uuid.uuid4())
        try:
            open(uuid_file, 'w').write(uuid_str)
        except:
            log.warning('Cannot store token, Please add write permissions to file:' + uuid_file)
            uuid_str = uuid_str + '.NO_WRITE'

    file_path = Path(mdb_file)
    if file_path.is_file():
        token = open(mdb_file).read()
    else:
github mindsdb / mindsdb / mindsdb / libs / controllers / transaction.py View on Github external
def save_metadata(self):
        fn = os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.lmd['name'] + '_light_model_metadata.pickle')
        self.lmd['updated_at'] = str(datetime.datetime.now())
        try:
            with open(fn, 'wb') as fp:
                pickle.dump(self.lmd, fp,protocol=pickle.HIGHEST_PROTOCOL)
        except:
            self.log.error(f'Could not save mindsdb heavy metadata in the file: {fn}')

        fn = os.path.join(CONFIG.MINDSDB_STORAGE_PATH, self.hmd['name'] + '_heavy_model_metadata.pickle')
        save_hmd = {}
        null_out_fields = ['test_from_data', 'from_data']
        for k in null_out_fields:
            save_hmd[k] = None


        for k in self.hmd:
            if k not in null_out_fields:
github mindsdb / mindsdb / mindsdb / libs / controllers / transaction.py View on Github external
if self.lmd['model_backend'] == 'ludwig':
                self.lmd['is_active'] = True
                self.model_backend = LudwigBackend(self)
                self.model_backend.train()
                self.lmd['is_active'] = False


            self.lmd['train_end_at'] = str(datetime.datetime.now())

            self._call_phase_module('ModelAnalyzer')

            with open(CONFIG.MINDSDB_STORAGE_PATH + '/' + self.lmd['name'] + '_light_model_metadata.pickle', 'wb') as fp:
                self.lmd['updated_at'] = str(datetime.datetime.now())
                pickle.dump(self.lmd, fp)

            with open(CONFIG.MINDSDB_STORAGE_PATH + '/' + self.hmd['name'] + '_heavy_model_metadata.pickle', 'wb') as fp:
                # Don't save data for now
                self.hmd['from_data'] = None
                self.hmd['test_from_data'] = None
                # Don't save data for now
                pickle.dump(self.hmd, fp)

            return

        except Exception as e:
            self.lmd['is_active'] = False
            self.lmd['current_phase'] = MODEL_STATUS_ERROR
            self.lmd['error_msg'] = traceback.print_exc()
            self.log.error(str(e))
            raise e
github mindsdb / mindsdb / mindsdb / libs / controllers / predictor.py View on Github external
try:
            lmd['ludwig_data']['ludwig_save_path'] = lmd['ludwig_data']['ludwig_save_path'].replace(old_model_name, new_model_name)
            renamed_one_backend = True
        except:
            pass

        try:
            lmd['lightwood_data']['save_path'] = lmd['lightwood_data']['save_path'].replace(old_model_name, new_model_name)
            renamed_one_backend = True
        except:
            pass

        if not renamed_one_backend:
            return False

        with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, new_model_name + '_light_model_metadata.pickle'), 'wb') as fp:
            pickle.dump(lmd, fp,protocol=pickle.HIGHEST_PROTOCOL)

        with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, new_model_name + '_heavy_model_metadata.pickle'), 'wb') as fp:
            pickle.dump(hmd, fp,protocol=pickle.HIGHEST_PROTOCOL)





        os.remove(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, old_model_name + '_light_model_metadata.pickle'))
        os.remove(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, old_model_name + '_heavy_model_metadata.pickle'))
        return True
github mindsdb / mindsdb / mindsdb / libs / controllers / predictor.py View on Github external
def get_model_data(self, model_name, lmd=None):
        if lmd is None:
            with open(os.path.join(CONFIG.MINDSDB_STORAGE_PATH, f'{model_name}_light_model_metadata.pickle'), 'rb') as fp:
                lmd = pickle.load(fp)
        # ADAPTOR CODE
        amd = {}

        if lmd['current_phase'] == MODEL_STATUS_TRAINED:
            amd['status'] = 'complete'
        elif lmd['current_phase'] == MODEL_STATUS_ERROR:
            amd['status'] = 'error'
        else:
            amd['status'] = 'training'

        # Shared keys
        for k in ['name', 'version', 'is_active', 'data_source', 'predict', 'current_phase',
        'train_end_at', 'updated_at', 'created_at','data_preparation', 'validation_set_accuracy']:
            if k == 'predict':
                amd[k] = lmd['predict_columns']
github mindsdb / mindsdb / mindsdb / libs / helpers / general_helpers.py View on Github external
def check_for_updates():
    """
    Check for updates of mindsdb
    it will ask the mindsdb server if there are new versions, if there are it will log a message

    :return: None
    """

    # tmp files
    uuid_file = CONFIG.MINDSDB_STORAGE_PATH + '/../uuid.mdb_base'
    mdb_file = CONFIG.MINDSDB_STORAGE_PATH + '/start.mdb_base'

    uuid_file_path = Path(uuid_file)
    if uuid_file_path.is_file():
        uuid_str = open(uuid_file).read()
    else:
        uuid_str = str(uuid.uuid4())
        try:
            open(uuid_file, 'w').write(uuid_str)
        except:
            log.warning('Cannot store token, Please add write permissions to file:' + uuid_file)
            uuid_str = uuid_str + '.NO_WRITE'

    file_path = Path(mdb_file)
    if file_path.is_file():
        token = open(mdb_file).read()