Examples with HistoryEventProto org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto used on opensource projects

Example 1 with HistoryEventProto

use of org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto in project tez by apache.

the class TestHistoryEventProtoConverter method testConvertAppLaunchedEvent.

@Test(timeout = 5000)
public void testConvertAppLaunchedEvent() {
    long launchTime = random.nextLong();
    long submitTime = random.nextLong();
    Configuration conf = new Configuration(false);
    conf.set("foo", "bar");
    conf.set("applicationId", "1234");
    MockVersionInfo mockVersionInfo = new MockVersionInfo();
    AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime, submitTime, user, conf, mockVersionInfo);
    HistoryEventProto proto = converter.convert(event);
    assertCommon(proto, HistoryEventType.APP_LAUNCHED, launchTime, EntityTypes.TEZ_APPLICATION, null, user, 3);
    assertEventData(proto, ATSConstants.CONFIG, null);
    assertEventData(proto, ATSConstants.TEZ_VERSION, null);
    assertEventData(proto, ATSConstants.DAG_AM_WEB_SERVICE_VERSION, AMWebController.VERSION);
}

Example 2 with HistoryEventProto

use of org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto in project tez by apache.

the class TestProtoHistoryLoggingService method assertEventsRead.

private void assertEventsRead(ProtoMessageReader<HistoryEventProto> reader, List<HistoryEventProto> protos, int start, int finish) throws Exception {
    for (int i = start; i < finish; ++i) {
        try {
            HistoryEventProto evt = reader.readEvent();
            Assert.assertEquals(protos.get(i), evt);
        } catch (EOFException e) {
            Assert.fail("Unexpected eof");
        }
    }
    try {
        HistoryEventProto evt = reader.readEvent();
        Assert.assertNull(evt);
    } catch (EOFException e) {
    // Expected.
    }
}

Example 3 with HistoryEventProto

use of org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto in project tez by apache.

the class TestProtoHistoryLoggingService method testService.

@Test
public void testService() throws Exception {
    ProtoHistoryLoggingService service = createService(false);
    service.start();
    TezDAGID dagId = TezDAGID.getInstance(appId, 0);
    List<HistoryEventProto> protos = new ArrayList<>();
    for (DAGHistoryEvent event : makeHistoryEvents(dagId, service)) {
        protos.add(new HistoryEventProtoConverter().convert(event.getHistoryEvent()));
        service.handle(event);
    }
    service.stop();
    TezProtoLoggers loggers = new TezProtoLoggers();
    Assert.assertTrue(loggers.setup(service.getConfig(), clock));
    // Verify dag events are logged.
    DatePartitionedLogger<HistoryEventProto> dagLogger = loggers.getDagEventsLogger();
    Path dagFilePath = dagLogger.getPathForDate(LocalDate.ofEpochDay(0), dagId + "_1");
    ProtoMessageReader<HistoryEventProto> reader = dagLogger.getReader(dagFilePath);
    assertEventsRead(reader, protos, 1, protos.size());
    reader.close();
    // Verify app events are logged.
    DatePartitionedLogger<HistoryEventProto> appLogger = loggers.getAppEventsLogger();
    Path appFilePath = appLogger.getPathForDate(LocalDate.ofEpochDay(0), attemptId.toString());
    ProtoMessageReader<HistoryEventProto> appReader = appLogger.getReader(appFilePath);
    long appOffset = appReader.getOffset();
    Assert.assertEquals(protos.get(0), appReader.readEvent());
    appReader.close();
    // Verify manifest events are logged.
    DatePartitionedLogger<ManifestEntryProto> manifestLogger = loggers.getManifestEventsLogger();
    Path manifestFilePath = manifestLogger.getPathForDate(LocalDate.ofEpochDay(0), attemptId.toString());
    ProtoMessageReader<ManifestEntryProto> reader2 = manifestLogger.getReader(manifestFilePath);
    ManifestEntryProto manifest = reader2.readEvent();
    Assert.assertEquals(appId.toString(), manifest.getAppId());
    Assert.assertEquals(dagId.toString(), manifest.getDagId());
    Assert.assertEquals(dagFilePath.toString(), manifest.getDagFilePath());
    Assert.assertEquals(appFilePath.toString(), manifest.getAppFilePath());
    Assert.assertEquals(appOffset, manifest.getAppLaunchedEventOffset());
    // Verify offsets in manifest logger.
    reader = dagLogger.getReader(new Path(manifest.getDagFilePath()));
    reader.setOffset(manifest.getDagSubmittedEventOffset());
    HistoryEventProto evt = reader.readEvent();
    Assert.assertNotNull(evt);
    Assert.assertEquals(HistoryEventType.DAG_SUBMITTED.name(), evt.getEventType());
    reader.setOffset(manifest.getDagFinishedEventOffset());
    evt = reader.readEvent();
    Assert.assertNotNull(evt);
    Assert.assertEquals(HistoryEventType.DAG_FINISHED.name(), evt.getEventType());
    reader.close();
    // Verify manifest file scanner.
    DagManifesFileScanner scanner = new DagManifesFileScanner(manifestLogger);
    Assert.assertEquals(manifest, scanner.getNext());
    Assert.assertNull(scanner.getNext());
    scanner.close();
}

Example 4 with HistoryEventProto

use of org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto in project tez by apache.

the class TestHistoryEventProtoConverter method testConvertAMStartedEvent.

@Test(timeout = 5000)
public void testConvertAMStartedEvent() {
    long startTime = random.nextLong();
    AMStartedEvent event = new AMStartedEvent(applicationAttemptId, startTime, user);
    HistoryEventProto proto = converter.convert(event);
    assertCommon(proto, HistoryEventType.AM_STARTED, startTime, EntityTypes.TEZ_APPLICATION, applicationAttemptId, user, 0);
}

Example 5 with HistoryEventProto

use of org.apache.tez.dag.history.logging.proto.HistoryLoggerProtos.HistoryEventProto in project tez by apache.

the class TestHistoryEventProtoConverter method testConvertTaskFinishedEvent.

@Test(timeout = 5000)
public void testConvertTaskFinishedEvent() {
    String vertexName = "testVertexName";
    long startTime = random.nextLong();
    long finishTime = random.nextLong();
    TaskState state = TaskState.values()[random.nextInt(TaskState.values().length)];
    String diagnostics = "diagnostics message";
    TezCounters counters = new TezCounters();
    TaskFinishedEvent event = new TaskFinishedEvent(tezTaskID, vertexName, startTime, finishTime, tezTaskAttemptID, state, diagnostics, counters, 3);
    HistoryEventProto proto = converter.convert(event);
    assertCommon(proto, HistoryEventType.TASK_FINISHED, finishTime, EntityTypes.TEZ_TASK_ID, null, null, 6);
    assertEventData(proto, ATSConstants.STATUS, state.name());
    assertEventData(proto, ATSConstants.TIME_TAKEN, String.valueOf(finishTime - startTime));
    assertEventData(proto, ATSConstants.SUCCESSFUL_ATTEMPT_ID, tezTaskAttemptID.toString());
    assertEventData(proto, ATSConstants.NUM_FAILED_TASKS_ATTEMPTS, "3");
    assertEventData(proto, ATSConstants.DIAGNOSTICS, diagnostics);
    assertEventData(proto, ATSConstants.COUNTERS, null);
}