Examples with IOMetrics org.apache.flink.runtime.executiongraph.IOMetrics used on opensource projects

Example 1 with IOMetrics

use of org.apache.flink.runtime.executiongraph.IOMetrics in project flink by apache.

the class MutableIOMetrics method addIOMetrics.

/**
	 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
	 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
	 * used to retrieve the required metrics.
	 * 
	 * @param attempt Attempt whose IO metrics should be added
	 * @param fetcher MetricFetcher to retrieve metrics for running jobs
	 * @param jobID JobID to which the attempt belongs
	 * @param taskID TaskID to which the attempt belongs
	 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
    if (attempt.getState().isTerminal()) {
        IOMetrics ioMetrics = attempt.getIOMetrics();
        if (ioMetrics != null) {
            // execAttempt is already finished, use final metrics stored in ExecutionGraph
            this.numBytesInLocal += ioMetrics.getNumBytesInLocal();
            this.numBytesInRemote += ioMetrics.getNumBytesInRemote();
            this.numBytesOut += ioMetrics.getNumBytesOut();
            this.numRecordsIn += ioMetrics.getNumRecordsIn();
            this.numRecordsOut += ioMetrics.getNumRecordsOut();
        }
    } else {
        // execAttempt is still running, use MetricQueryService instead
        if (fetcher != null) {
            fetcher.update();
            MetricStore.SubtaskMetricStore metrics = fetcher.getMetricStore().getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
            if (metrics != null) {
                this.numBytesInLocal += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL, "0"));
                this.numBytesInRemote += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE, "0"));
                this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT, "0"));
                this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN, "0"));
                this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT, "0"));
            }
        }
    }
}

Example 2 with IOMetrics

use of org.apache.flink.runtime.executiongraph.IOMetrics in project flink by apache.

the class JobDetailsHandlerTest method compareJobDetails.

private static void compareJobDetails(AccessExecutionGraph originalJob, String json) throws IOException {
    JsonNode result = ArchivedJobGenerationUtils.mapper.readTree(json);
    Assert.assertEquals(originalJob.getJobID().toString(), result.get("jid").asText());
    Assert.assertEquals(originalJob.getJobName(), result.get("name").asText());
    Assert.assertEquals(originalJob.isStoppable(), result.get("isStoppable").asBoolean());
    Assert.assertEquals(originalJob.getState().name(), result.get("state").asText());
    Assert.assertEquals(originalJob.getStatusTimestamp(JobStatus.CREATED), result.get("start-time").asLong());
    Assert.assertEquals(originalJob.getStatusTimestamp(originalJob.getState()), result.get("end-time").asLong());
    Assert.assertEquals(originalJob.getStatusTimestamp(originalJob.getState()) - originalJob.getStatusTimestamp(JobStatus.CREATED), result.get("duration").asLong());
    JsonNode timestamps = result.get("timestamps");
    for (JobStatus status : JobStatus.values()) {
        Assert.assertEquals(originalJob.getStatusTimestamp(status), timestamps.get(status.name()).asLong());
    }
    ArrayNode tasks = (ArrayNode) result.get("vertices");
    int x = 0;
    for (AccessExecutionJobVertex expectedTask : originalJob.getVerticesTopologically()) {
        JsonNode task = tasks.get(x);
        Assert.assertEquals(expectedTask.getJobVertexId().toString(), task.get("id").asText());
        Assert.assertEquals(expectedTask.getName(), task.get("name").asText());
        Assert.assertEquals(expectedTask.getParallelism(), task.get("parallelism").asInt());
        Assert.assertEquals(expectedTask.getAggregateState().name(), task.get("status").asText());
        Assert.assertEquals(3, task.get("start-time").asLong());
        Assert.assertEquals(5, task.get("end-time").asLong());
        Assert.assertEquals(2, task.get("duration").asLong());
        JsonNode subtasksPerState = task.get("tasks");
        Assert.assertEquals(0, subtasksPerState.get(ExecutionState.CREATED.name()).asInt());
        Assert.assertEquals(0, subtasksPerState.get(ExecutionState.SCHEDULED.name()).asInt());
        Assert.assertEquals(0, subtasksPerState.get(ExecutionState.DEPLOYING.name()).asInt());
        Assert.assertEquals(0, subtasksPerState.get(ExecutionState.RUNNING.name()).asInt());
        Assert.assertEquals(1, subtasksPerState.get(ExecutionState.FINISHED.name()).asInt());
        Assert.assertEquals(0, subtasksPerState.get(ExecutionState.CANCELING.name()).asInt());
        Assert.assertEquals(0, subtasksPerState.get(ExecutionState.CANCELED.name()).asInt());
        Assert.assertEquals(0, subtasksPerState.get(ExecutionState.FAILED.name()).asInt());
        long expectedNumBytesIn = 0;
        long expectedNumBytesOut = 0;
        long expectedNumRecordsIn = 0;
        long expectedNumRecordsOut = 0;
        for (AccessExecutionVertex vertex : expectedTask.getTaskVertices()) {
            IOMetrics ioMetrics = vertex.getCurrentExecutionAttempt().getIOMetrics();
            expectedNumBytesIn += ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote();
            expectedNumBytesOut += ioMetrics.getNumBytesOut();
            expectedNumRecordsIn += ioMetrics.getNumRecordsIn();
            expectedNumRecordsOut += ioMetrics.getNumRecordsOut();
        }
        JsonNode metrics = task.get("metrics");
        Assert.assertEquals(expectedNumBytesIn, metrics.get("read-bytes").asLong());
        Assert.assertEquals(expectedNumBytesOut, metrics.get("write-bytes").asLong());
        Assert.assertEquals(expectedNumRecordsIn, metrics.get("read-records").asLong());
        Assert.assertEquals(expectedNumRecordsOut, metrics.get("write-records").asLong());
        x++;
    }
    Assert.assertEquals(1, tasks.size());
    JsonNode statusCounts = result.get("status-counts");
    Assert.assertEquals(0, statusCounts.get(ExecutionState.CREATED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.SCHEDULED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.DEPLOYING.name()).asInt());
    Assert.assertEquals(1, statusCounts.get(ExecutionState.RUNNING.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.FINISHED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.CANCELING.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.CANCELED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.FAILED.name()).asInt());
    Assert.assertEquals(ArchivedJobGenerationUtils.mapper.readTree(originalJob.getJsonPlan()), result.get("plan"));
}

Example 3 with IOMetrics

use of org.apache.flink.runtime.executiongraph.IOMetrics in project flink by apache.

the class JobVertexTaskManagersHandlerTest method compareVertexTaskManagers.

private static void compareVertexTaskManagers(AccessExecutionJobVertex originalTask, AccessExecutionVertex originalSubtask, String json) throws IOException {
    JsonNode result = ArchivedJobGenerationUtils.mapper.readTree(json);
    Assert.assertEquals(originalTask.getJobVertexId().toString(), result.get("id").asText());
    Assert.assertEquals(originalTask.getName(), result.get("name").asText());
    Assert.assertTrue(result.get("now").asLong() > 0);
    ArrayNode taskmanagers = (ArrayNode) result.get("taskmanagers");
    JsonNode taskManager = taskmanagers.get(0);
    TaskManagerLocation location = originalSubtask.getCurrentAssignedResourceLocation();
    String expectedLocationString = location.getHostname() + ':' + location.dataPort();
    Assert.assertEquals(expectedLocationString, taskManager.get("host").asText());
    Assert.assertEquals(ExecutionState.FINISHED.name(), taskManager.get("status").asText());
    Assert.assertEquals(3, taskManager.get("start-time").asLong());
    Assert.assertEquals(5, taskManager.get("end-time").asLong());
    Assert.assertEquals(2, taskManager.get("duration").asLong());
    JsonNode statusCounts = taskManager.get("status-counts");
    Assert.assertEquals(0, statusCounts.get(ExecutionState.CREATED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.SCHEDULED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.DEPLOYING.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.RUNNING.name()).asInt());
    Assert.assertEquals(1, statusCounts.get(ExecutionState.FINISHED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.CANCELING.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.CANCELED.name()).asInt());
    Assert.assertEquals(0, statusCounts.get(ExecutionState.FAILED.name()).asInt());
    long expectedNumBytesIn = 0;
    long expectedNumBytesOut = 0;
    long expectedNumRecordsIn = 0;
    long expectedNumRecordsOut = 0;
    for (AccessExecutionVertex vertex : originalTask.getTaskVertices()) {
        IOMetrics ioMetrics = vertex.getCurrentExecutionAttempt().getIOMetrics();
        expectedNumBytesIn += ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote();
        expectedNumBytesOut += ioMetrics.getNumBytesOut();
        expectedNumRecordsIn += ioMetrics.getNumRecordsIn();
        expectedNumRecordsOut += ioMetrics.getNumRecordsOut();
    }
    JsonNode metrics = taskManager.get("metrics");
    Assert.assertEquals(expectedNumBytesIn, metrics.get("read-bytes").asLong());
    Assert.assertEquals(expectedNumBytesOut, metrics.get("write-bytes").asLong());
    Assert.assertEquals(expectedNumRecordsIn, metrics.get("read-records").asLong());
    Assert.assertEquals(expectedNumRecordsOut, metrics.get("write-records").asLong());
}

Example 4 with IOMetrics

use of org.apache.flink.runtime.executiongraph.IOMetrics in project flink by apache.

the class TaskIOMetricGroupTest method testTaskIOMetricGroup.

@Test
public void testTaskIOMetricGroup() {
    TaskMetricGroup task = new UnregisteredTaskMetricsGroup();
    TaskIOMetricGroup taskIO = task.getIOMetricGroup();
    // test counter forwarding
    assertNotNull(taskIO.getNumRecordsInCounter());
    assertNotNull(taskIO.getNumRecordsOutCounter());
    Counter c1 = new SimpleCounter();
    c1.inc(32L);
    Counter c2 = new SimpleCounter();
    c2.inc(64L);
    taskIO.reuseRecordsInputCounter(c1);
    taskIO.reuseRecordsOutputCounter(c2);
    assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
    assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());
    // test IOMetrics instantiation
    taskIO.getNumBytesInLocalCounter().inc(100L);
    taskIO.getNumBytesInRemoteCounter().inc(150L);
    taskIO.getNumBytesOutCounter().inc(250L);
    IOMetrics io = taskIO.createSnapshot();
    assertEquals(32L, io.getNumRecordsIn());
    assertEquals(64L, io.getNumRecordsOut());
    assertEquals(100L, io.getNumBytesInLocal());
    assertEquals(150L, io.getNumBytesInRemote());
    assertEquals(250L, io.getNumBytesOut());
}