How to use the reader.FileReader.readLines function in reader

To help you get started, we’ve selected a few reader examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github JerryLead / SparkProfiler / src / python / plotter / HeapUsageAndGCPause2.py View on Github external
def initHeapUsageAndGCPause(self, gclogFile, gcpauseFile):
        fileLines = FileReader.readLines(gclogFile)
        for line in fileLines:
            if (line.strip != ""):
                heapUsage = self.parseUsage(line)
                if(line.startswith("[Young]")):
                    self.youngGen.append(heapUsage)
                elif(line.startswith("[Old]")):
                    self.oldGen.append(heapUsage)
                elif(line.startswith("[Metaspace]")):
                    self.metaGen.append(heapUsage)

        fileLines2 = FileReader.readLines(gcpauseFile)
        self.gcTotal["GCPause"] = []
        self.gcTimeLine["GCPause"] = []
        for line in fileLines2:
            if not line[0].isdigit():
                continue
github JerryLead / SparkProfiler / src / python / plotter / HeapUsageAndGCPause.py View on Github external
def initHeapUsageAndGCPause(self, gclogFile, gcpauseFile):
        fileLines = FileReader.readLines(gclogFile)
        for line in fileLines:
            if (line.strip != ""):
                heapUsage = self.parseUsage(line)
                if(line.startswith("[Young]")):
                    self.youngGen.append(heapUsage)
                elif(line.startswith("[Old]")):
                    self.oldGen.append(heapUsage)
                elif(line.startswith("[Metaspace]")):
                    self.metaGen.append(heapUsage)

        fileLines2 = FileReader.readLines(gcpauseFile)
        for line in fileLines2:
            if not line[0].isdigit():
                continue
            items = re.split(",", line)
            gc_type = items[-1]
github JerryLead / SparkProfiler / src / python / Join / JoinMemoryUsageGCWithMark.py View on Github external
def initHeapUsage(self, gclogFile, timeOffset):
        fileLines = FileReader.readLines(gclogFile)
        for line in fileLines:
            if (line.strip != ""):
                heapUsage = self.parseUsage(line, timeOffset)
                if(line.startswith("[Young]")):
                    self.youngGen.append(heapUsage)
                elif(line.startswith("[Old]")):
                    self.oldGen.append(heapUsage)
github JerryLead / SparkProfiler / src / python / plotter / TimeSeriesPlotter2.py View on Github external
def plotResourceUsage(topMetricsFile, slowestTasksDir, appName):
    fileLines = FileReader.readLines(topMetricsFile)

    isExecutorMetric = False
    isSlaveMetric = False

    executorTime = []
    executorCPU = []
    executorMemory = []

    slaveTime = []
    slaveCPU = []
    slaveMemory = []

    first_time = -1

    for line in fileLines:
        if (line.startswith("[Top Metrics][Executor")):
github JerryLead / SparkProfiler / src / python / analyzer / GroupAppMetricsAnalyzer.py View on Github external
def fillStatistics(self, metrics, statisticsDir, statisticsFiles, metricsMap, withMax):
        metricsTupleDict = {} # ["app.duration", ("app.duration", "Time (s)", 1000)]

        for tuple in metrics:
            metricsTupleDict[tuple[0]] = tuple
        for file in statisticsFiles:
            if file.endswith("stat.txt"): # [RDDJoin-CMS-1-7G-stat.txt, RDDJoin-CMS-2-14G-stat.txt, ...]
                # [app.duration] mean = 224339.20, stdVar = 8311.91, median = 225233.00, min = 211999.00, quantile25 = 216682.50, quantile75 = 231549.00, max = 233837.00
                # -------------------------------------------------------------------[Stage 0]-------------------------------------------------------------------
                # [stage0.duration] mean = 42360.60, stdVar = 4069.63, median = 41404.00, min = 37094.00, quantile25 = 38942.50, quantile75 = 46257.00, max = 47801.00
                # [stage0.inputBytes] mean = 8588671743.00, stdVar = 0.00, median = 8588671743.00, min = 8588671743.00, quantile25 = 8588671743.00, quantile75 = 8588671743.00, max = 8588671743.00
                # [stage0.inputRecords] mean = 66000000.00, stdVar = 0.00, median = 66000000.00, min = 66000000.00, quantile25 = 66000000.00, quantile75 = 66000000.00, max = 66000000.00
                for line in FileReader.readLines(os.path.join(statisticsDir, file)):
                    metricName = line[line.find('[') + 1: line.find(']')]
                    if metricsTupleDict.has_key(metricName):
                        if metricsMap.has_key(metricName):
                            metricsMap[metricName].addStatistics(line, file, withMax)
                        else:
                            statistics = st.BoxPlotStatistics(metricsTupleDict[metricName])
                            statistics.addStatistics(line, file, withMax)
                            metricsMap[metricName] = statistics

        # Fill the NaA values
        for metricName, statistics in metricsMap.items():
            statistics.checkAndFillNulls()
github JerryLead / SparkProfiler / src / python / plotter / HeapUsage.py View on Github external
def initHeapUsage(self, gclogFile):
        fileLines = FileReader.readLines(gclogFile)
        for line in fileLines:
            if (line.strip != ""):
                heapUsage = self.parseUsage(line)
                if(line.startswith("[Young]")):
                    self.youngGen.append(heapUsage)
                elif(line.startswith("[Old]")):
                    self.oldGen.append(heapUsage)
github JerryLead / SparkProfiler / src / python / PageRank / PageRankMemoryUsageAndGCPauseAxis.py View on Github external
def initHeapUsage(self, gclogFile, timeOffset):
        fileLines = FileReader.readLines(gclogFile)
        for line in fileLines:
            if (line.strip != ""):
                heapUsage = self.parseUsage(line, timeOffset)
                if(line.startswith("[Young]")):
                    self.youngGen.append(heapUsage)
                elif(line.startswith("[Old]")):
                    self.oldGen.append(heapUsage)
github JerryLead / SparkProfiler / src / python / Join / JoinMemoryUsageAndGCPauseAxis.py View on Github external
def initHeapUsage(self, gclogFile, timeOffset):
        fileLines = FileReader.readLines(gclogFile)
        for line in fileLines:
            if (line.strip != ""):
                heapUsage = self.parseUsage(line, timeOffset)
                if(line.startswith("[Young]")):
                    self.youngGen.append(heapUsage)
                elif(line.startswith("[Old]")):
                    self.oldGen.append(heapUsage)
github JerryLead / SparkProfiler / src / python / analyzer / TaskMetricsCorrelationAnalyzer.py View on Github external
executor = ""
                if (file.find("CMS") != -1):
                    GC = "CMS"
                elif (file.find("G1") != -1):
                    GC = "G1"
                elif (file.find("Parallel") != -1):
                    GC = "Parallel"

                if (file.find("1-7G") != -1):
                    executor = "E1"
                elif (file.find("2-14G") != -1):
                    executor = "E2"
                elif (file.find("4-28G") != -1):
                    executor = "E4"

                for line in FileReader.readLines(os.path.join(self.taskInfoDir, file)):
                    if line.startswith('[appName'):
                        appName = line[line.find('=') + 2: line.find(']')]
                    elif line.startswith('[appId'):
                        appId = line[line.find('=') + 2: line.find(']')]
                    elif line.startswith('[stageId'):
                        stageId = int(line[line.find('=') + 2: line.find(']')])
                    elif line.startswith('['):
                        metricName = line[line.find('task') + 5: line.find(']')]
                        metricValue = float(line[line.find(']') + 2:])
                        if (metricName == 'index'):
                            taskAttempt = ta.TaskAttempt(appId, appName, stageId, int(metricValue), GC, executor)
                        taskAttempt.set(metricName, metricValue)
                        self.taskList.append(taskAttempt)
github JerryLead / SparkProfiler / src / python / plotter / TimeSeriesPlotter.py View on Github external
def plotResourceUsage(topMetricsFile, slowestTasksDir, appName):
    fileLines = FileReader.readLines(topMetricsFile)

    isExecutorMetric = False
    isSlaveMetric = False

    executorTime = []
    executorCPU = []
    executorMemory = []

    slaveTime = []
    slaveCPU = []
    slaveMemory = []

    for line in fileLines:
        if (line.startswith("[Top Metrics][Executor")):
            isExecutorMetric = True
        elif (line.startswith("[Top Metrics][aliSlave")):