How to use the carbonate.fill.fill_archives function in carbonate

To help you get started, we’ve selected a few carbonate examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github graphite-project / carbonate / tests / test_fill.py View on Github external
three_quarter =  points_number * 3 // 4

        # fills a fourth of data, from 2/4th to 3/4th
        fill_archives(self.db, dst_db, time.time()-quarter, time.time()-half)
        quarter_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        expected = [None]*half + complete[half:three_quarter] + [None]*quarter
        self.assertEqual(expected, quarter_filled_data)
        # Fetching data older than start forces the use of the second level of aggregation
        # We get a first empty cell and then
        quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1]
        average_l1 = _average(quarter_filled_data)
        average_l2 = _average(quarter_filled_data_l2)
        self.assertEqual(average_l1, average_l2)

        # fills a half of data, from 2/4th to 4/4th
        fill_archives(self.db, dst_db, time.time(), time.time()-half)
        half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        expected = [None]*half + complete[half:]
        self.assertEqual(expected, half_filled_data)

        # Explicitly passes the default value of endAt=now (excluded)
        fill_archives(self.db, dst_db, time.time(), endAt=0)
        filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        self.assertEqual(complete[:-1], filled_data[:-1])
        self.assertEqual(filled_data[-1], None)
github graphite-project / carbonate / tests / test_fill.py View on Github external
empty_data = []

        end = int(time.time()) + seconds_per_point
        start = end - (points_number * seconds_per_point)
        times = range(start, end, seconds_per_point)

        complete_data = zip(times, complete)
        self._createdb(self.db, schema, complete_data)
        self._createdb(dst_db, schema, empty_data)

        quarter = points_number // 4
        half = points_number // 2
        three_quarter =  points_number * 3 // 4

        # fills a fourth of data, from 2/4th to 3/4th
        fill_archives(self.db, dst_db, time.time()-quarter, time.time()-half)
        quarter_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        expected = [None]*half + complete[half:three_quarter] + [None]*quarter
        self.assertEqual(expected, quarter_filled_data)
        # Fetching data older than start forces the use of the second level of aggregation
        # We get a first empty cell and then
        quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1]
        average_l1 = _average(quarter_filled_data)
        average_l2 = _average(quarter_filled_data_l2)
        self.assertEqual(average_l1, average_l2)

        # fills a half of data, from 2/4th to 4/4th
        fill_archives(self.db, dst_db, time.time(), time.time()-half)
        half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        expected = [None]*half + complete[half:]
        self.assertEqual(expected, half_filled_data)
github graphite-project / carbonate / tests / test_fill.py View on Github external
schema = [(1, 20)]
        data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
                11, 12, 13, 14, 15, 16, 17, 18, 19, 20]

        end = int(time.time()) + schema[0][0]
        start = end - (schema[0][1] * schema[0][0])
        times = range(start, end, schema[0][0])

        override_data = zip(times, data)

        startTime = time.time()
        self._createdb(self.db, schema)
        self._createdb(testdb, schema, override_data)

        fill_archives(self.db, testdb, startTime)

        filled_data = whisper.fetch(testdb, 0)
        self.assertEqual(data, filled_data[1])
github graphite-project / carbonate / tests / test_fill.py View on Github external
def test_fill_empty(self):
        testdb = "test-%s" % self.db
        self._removedb()

        try:
            os.unlink(testdb)
        except (IOError, OSError):
            pass

        schema = [(1, 20)]
        emptyData = []
        startTime = time.time()
        self._createdb(self.db, schema)
        self._createdb(testdb, schema, emptyData)

        fill_archives(self.db, testdb, startTime)

        original_data = whisper.fetch(self.db, 0)
        filled_data = whisper.fetch(testdb, 0)
        self.assertEqual(original_data, filled_data)
github graphite-project / carbonate / tests / test_fill.py View on Github external
self.assertEqual(expected, quarter_filled_data)
        # Fetching data older than start forces the use of the second level of aggregation
        # We get a first empty cell and then
        quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1]
        average_l1 = _average(quarter_filled_data)
        average_l2 = _average(quarter_filled_data_l2)
        self.assertEqual(average_l1, average_l2)

        # fills a half of data, from 2/4th to 4/4th
        fill_archives(self.db, dst_db, time.time(), time.time()-half)
        half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        expected = [None]*half + complete[half:]
        self.assertEqual(expected, half_filled_data)

        # Explicitly passes the default value of endAt=now (excluded)
        fill_archives(self.db, dst_db, time.time(), endAt=0)
        filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        self.assertEqual(complete[:-1], filled_data[:-1])
        self.assertEqual(filled_data[-1], None)
github graphite-project / carbonate / tests / test_fill.py View on Github external
6,
                None, None, None, None,
                11, 12, 13, 14, 15, 16, 17, 18, 19,
                None, # trailing missing data point
                ]

        end = int(time.time()) + schema[0][0]
        start = end - (schema[0][1] * schema[0][0])
        times = range(start, end, schema[0][0])

        original_data = [t for t in zip(times, original) if t[1] is not None]
        holes_data = [t for t in zip(times, holes) if t[1] is not None]
        self._createdb(self.db, schema, original_data)
        self._createdb(testdb, schema, holes_data)

        fill_archives(self.db, testdb, time.time())

        filled_data = whisper.fetch(testdb, 0)
        self.assertEqual(complete, filled_data[1])
github graphite-project / carbonate / carbonate / cli.py View on Github external
help='Whisper destination file')

    args = parser.parse_args()

    src = args.source
    dst = args.dest

    if not os.path.isfile(src):
        raise SystemExit('Source file not found.')

    if not os.path.isfile(dst):
        raise SystemExit('Destination file not found.')

    startFrom = time()

    fill_archives(src, dst, startFrom)
github graphite-project / carbonate / carbonate / cli.py View on Github external
help='Whisper destination file')

    args = parser.parse_args()

    src = args.source
    dst = args.dest

    if not os.path.isfile(src):
        raise SystemExit('Source file not found.')

    if not os.path.isfile(dst):
        raise SystemExit('Destination file not found.')

    startFrom = time()

    fill_archives(src, dst, startFrom)
github graphite-project / carbonate / carbonate / delete.py View on Github external
def deleteMetric(metric,
                 storage_dir='/opt/graphite/storage/whisper',
                 trash_dir="/opt/graphite/storage/whisper/trash"):
    oldMetric = metric_path(metric, storage_dir)
    newMetric = metric_path(metric, trash_dir)

    if os.path.isfile(oldMetric):
        try:
            os.makedirs(os.path.dirname(newMetric))
        except os.error:
            pass

        startFrom = time()
        try:
            fill_archives(oldMetric, newMetric, startFrom)
            os.unlink(oldMetric)
        except OSError, e:
            if e.errno != errno.ENOENT:
                raise
github graphite-project / carbonate / carbonate / sync.py View on Github external
def heal_metric(source, dest):
    try:
        with open(dest):
            try:
                fill_archives(source, dest, time())
            except CorruptWhisperFile as e:
                logging.warn("Overwriting corrupt file %s!" % dest)
                try:
                    os.makedirs(os.path.dirname(dest))
                except os.error:
                    pass
                try:
                    shutil.copyfile(source, dest)
                except IOError as e:
                    logging.warn("Failed to copy %s! %s" % (dest, e))
    except IOError:
        try:
            os.makedirs(os.path.dirname(dest))
        except os.error:
            pass
        try: