How to use the arctic.decorators.mongo_retry function in arctic

To help you get started, we’ve selected a few arctic examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github man-group / arctic / arctic / store / version_store.py View on Github external
def _write_audit(self, user, message, changed_version):
        """
        Creates an audit entry, which is much like a snapshot in that
        it references versions and provides some history of the changes made.
        """
        audit = {'_id': bson.ObjectId(),
                 'user': user,
                 'message': message,
                 'symbol': changed_version.symbol
                 }
        orig_version = changed_version.orig_version.version
        new_version = changed_version.new_version.version
        audit['orig_v'] = orig_version
        audit['new_v'] = new_version
        # Update the versions to contain the audit
        mongo_retry(self._versions.update_many)({'symbol': changed_version.symbol,
                                                 'version': {'$in': [orig_version, new_version]}
                                                 },
                                                {'$addToSet': {'parent': audit['_id']}})
        # Create the audit entry
        mongo_retry(self._audit.insert_one)(audit)
github man-group / arctic / arctic / chunkstore / chunkstore.py View on Github external
SEGMENT: chunk[SEGMENT]},
                                                 {'$set': chunk}, upsert=True))
                else:
                    # already exists, dont need to update in mongo
                    previous_shas.remove(chunk[SHA])

        if ops:
            self._collection.bulk_write(ops, ordered=False)
        if meta_ops:
            self._mdata.bulk_write(meta_ops, ordered=False)

        doc[CHUNK_COUNT] = chunk_count
        doc[APPEND_COUNT] = 0

        if previous_shas:
            mongo_retry(self._collection.delete_many)({SYMBOL: symbol, SHA: {'$in': list(previous_shas)}})

        mongo_retry(self._symbols.update_one)({SYMBOL: symbol},
                                              {'$set': doc},
                                              upsert=True)
        if audit is not None:
            audit['symbol'] = symbol
            audit['action'] = 'write'
            audit['chunks'] = chunk_count
            self._audit.insert_one(audit)
github man-group / arctic / arctic / store / metadata_store.py View on Github external
    @mongo_retry
    def read_history(self, symbol):
        """
        Return all metadata saved for `symbol`

        Parameters
        ----------
        symbol : `str`
            symbol name for the item

        Returns
        -------
        pandas.DateFrame containing timestamps and metadata entries
        """
        find = self.find({'symbol': symbol}, sort=[('start_time', pymongo.ASCENDING)])
        times = []
        entries = []
github man-group / arctic / arctic / store / metadata_store.py View on Github external
if old_metadata['start_time'] <= start_time:
                raise ValueError('start_time={} is later than the first metadata @{}'.format(start_time,
                                                                                             old_metadata['start_time']))
            if old_metadata['metadata'] == metadata:
                self.find_one_and_update({'symbol': symbol}, {'$set': {'start_time': start_time}},
                                         sort=[('start_time', pymongo.ASCENDING)])
                old_metadata['start_time'] = start_time
                return old_metadata
            end_time = old_metadata.get('start_time')
        else:
            end_time = None

        document = {'_id': bson.ObjectId(), 'symbol': symbol, 'metadata': metadata, 'start_time': start_time}
        if end_time is not None:
            document['end_time'] = end_time
        mongo_retry(self.insert_one)(document)

        logger.debug('Finished writing metadata for %s', symbol)
        return document
github man-group / arctic / arctic / store / bson_store.py View on Github external
def initialize_library(cls, arctic_lib, **kwargs):
        logger.info("Creating BSONStore without sharding. Use BSONStore.enable_sharding to "
                    "enable sharding for large amounts of data.")
        c = arctic_lib.get_top_level_collection()
        if c.name not in mongo_retry(c.database.list_collection_names)():
            mongo_retry(c.database.create_collection)(c.name)
        else:
            logger.warning("Collection %s already exists", c.name)
github man-group / arctic / arctic / store / bson_store.py View on Github external
    @mongo_retry
    def find_one_and_update(self, filter, update, **kwargs):
        """
        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one_and_update
        """
        self._arctic_lib.check_quota()
        return self._collection.find_one_and_update(filter, update, **kwargs)
github man-group / arctic / arctic / chunkstore / chunkstore.py View on Github external
audit: dict
            audit information
        """

        sym = self._get_symbol_info(from_symbol)
        if not sym:
            raise NoDataFoundException('No data found for %s' % (from_symbol))

        if self._get_symbol_info(to_symbol) is not None:
            raise Exception('Symbol %s already exists' % (to_symbol))

        mongo_retry(self._collection.update_many)({SYMBOL: from_symbol},
                                                  {'$set': {SYMBOL: to_symbol}})
        mongo_retry(self._symbols.update_one)({SYMBOL: from_symbol},
                                              {'$set': {SYMBOL: to_symbol}})
        mongo_retry(self._mdata.update_many)({SYMBOL: from_symbol},
                                             {'$set': {SYMBOL: to_symbol}})
        mongo_retry(self._audit.update_many)({'symbol': from_symbol},
                                             {'$set': {'symbol': to_symbol}})
        if audit is not None:
            audit['symbol'] = to_symbol
            audit['action'] = 'symbol rename'
            audit['old_symbol'] = from_symbol
            self._audit.insert_one(audit)
github man-group / arctic / arctic / store / bson_store.py View on Github external
    @mongo_retry
    def insert_one(self, document, **kwargs):
        """
        See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_one
        """
        self._arctic_lib.check_quota()
        return self._collection.insert_one(document, **kwargs)
github man-group / arctic / arctic / arctic.py View on Github external
    @mongo_retry
    def set_library_metadata(self, field, value):
        self._library_coll[self.arctic.METADATA_COLL].update_one({'_id': self.arctic.METADATA_DOC_ID},
                                                                 {'$set': {field: value}}, upsert=True)
github man-group / arctic / arctic / store / version_store.py View on Github external
# and the new version's "base_version_id" won't be referenced by any segments.
        # Do a naive check for concurrent mods
        lastv_seqn = self._last_version_seqnum(symbol)
        if lastv_seqn != new_version['version']:
            raise OperationFailure("The symbol {} has been modified concurrently ({} != {})".format(
                symbol, lastv_seqn, new_version['version']))

        # Insert the new version into the version DB
        # (must come before the pruning, otherwise base version won't be preserved)
        self._insert_version(new_version)

        # Check if in the meanwhile the reference version (based on which we updated incrementally) has been removed
        last_look = self._versions.find_one({'_id': reference_version['_id']})
        if last_look is None or last_look.get('deleted'):
            # Revert the change
            mongo_retry(self._versions.delete_one)({'_id': new_version['_id']})
            # Indicate the failure
            raise OperationFailure("Failed to write metadata for symbol %s. "
                                   "The previous version (%s, %d) has been removed during the update" %
                                   (symbol, str(reference_version['_id']), reference_version['version']))

        if prune_previous_version and reference_version:
            self._prune_previous_versions(symbol, new_version_shas=new_version.get(FW_POINTERS_REFS_KEY))

        logger.debug('Finished updating versions with new metadata for %s', symbol)

        return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=new_version['version'],
                             metadata=new_version.get('metadata'), data=None,
                             host=self._arctic_lib.arctic.mongo_host)