Examples with MetricsSnapshotSerdeV2 org.apache.samza.serializers.MetricsSnapshotSerdeV2 used on opensource projects

Example 1 with MetricsSnapshotSerdeV2

use of org.apache.samza.serializers.MetricsSnapshotSerdeV2 in project samza by apache.

the class DiagnosticsUtil method writeMetadataFile.

// Write a file in the samza.log.dir named {exec-env-container-id}.metadata that contains
// metadata about the container such as containerId, jobName, jobId, hostname, timestamp, version info, and others.
// The file contents are serialized using {@link JsonSerde}.
public static void writeMetadataFile(String jobName, String jobId, String containerId, Optional<String> execEnvContainerId, Config config) {
    Optional<File> metadataFile = JobConfig.getMetadataFile(execEnvContainerId.orElse(null));
    if (metadataFile.isPresent()) {
        MetricsHeader metricsHeader = new MetricsHeader(jobName, jobId, "samza-container-" + containerId, execEnvContainerId.orElse(""), LocalContainerRunner.class.getName(), Util.getTaskClassVersion(config), Util.getSamzaVersion(), Util.getLocalHost().getHostName(), System.currentTimeMillis(), System.currentTimeMillis());
        class MetadataFileContents {

            public final String version;

            public final String metricsSnapshot;

            public MetadataFileContents(String version, String metricsSnapshot) {
                this.version = version;
                this.metricsSnapshot = metricsSnapshot;
            }
        }
        MetricsSnapshot metricsSnapshot = new MetricsSnapshot(metricsHeader, new Metrics());
        MetadataFileContents metadataFileContents = new MetadataFileContents("1", new String(new MetricsSnapshotSerdeV2().toBytes(metricsSnapshot)));
        new FileUtil().writeToTextFile(metadataFile.get(), new String(new JsonSerde<>().toBytes(metadataFileContents)), false);
    } else {
        log.info("Skipping writing metadata file.");
    }
}

Example 2 with MetricsSnapshotSerdeV2

use of org.apache.samza.serializers.MetricsSnapshotSerdeV2 in project samza by apache.

the class TestMetricsSnapshotSerdeV2 method testDeserializeRawEmptySamzaEpochIdInHeader.

/**
 * Helps for verifying compatibility when schemas evolve.
 */
@Test
public void testDeserializeRawEmptySamzaEpochIdInHeader() {
    SamzaException samzaException = new SamzaException("this is a samza exception", new RuntimeException("cause"));
    MetricsSnapshot metricsSnapshot = metricsSnapshot(samzaException, false);
    MetricsSnapshotSerdeV2 metricsSnapshotSerde = new MetricsSnapshotSerdeV2();
    assertEquals(metricsSnapshot, metricsSnapshotSerde.fromBytes(expectedSeralizedSnapshot(samzaException, false, false).getBytes(StandardCharsets.UTF_8)));
    assertEquals(metricsSnapshot, metricsSnapshotSerde.fromBytes(expectedSeralizedSnapshot(samzaException, false, true).getBytes(StandardCharsets.UTF_8)));
}

Example 3 with MetricsSnapshotSerdeV2

use of org.apache.samza.serializers.MetricsSnapshotSerdeV2 in project samza by apache.

the class TestDiagnosticsManager method testSecondPublishWithProcessorStopInSecondMessage.

@Test
public void testSecondPublishWithProcessorStopInSecondMessage() {
    // Across two successive run() invocations two messages should be published if stop events are added
    this.diagnosticsManager.start();
    this.diagnosticsManager.addProcessorStopEvent("0", EXECUTION_ENV_CONTAINER_ID, HOSTNAME, 102);
    this.diagnosticsManager.start();
    Assert.assertEquals("Two messages should have been published", 2, mockSystemProducer.getEnvelopeList().size());
    // Validate the first message
    OutgoingMessageEnvelope outgoingMessageEnvelope = mockSystemProducer.getEnvelopeList().get(0);
    validateMetricsHeader(outgoingMessageEnvelope, FIRST_SEND_TIME);
    validateOutgoingMessageEnvelope(outgoingMessageEnvelope);
    // Validate the second message's header
    outgoingMessageEnvelope = mockSystemProducer.getEnvelopeList().get(1);
    validateMetricsHeader(outgoingMessageEnvelope, SECOND_SEND_TIME);
    // Validate the second message's body (should be all empty except for the processor-stop-event)
    MetricsSnapshot metricsSnapshot = new MetricsSnapshotSerdeV2().fromBytes((byte[]) outgoingMessageEnvelope.getMessage());
    DiagnosticsStreamMessage diagnosticsStreamMessage = DiagnosticsStreamMessage.convertToDiagnosticsStreamMessage(metricsSnapshot);
    Assert.assertNull(diagnosticsStreamMessage.getContainerMb());
    Assert.assertNull(diagnosticsStreamMessage.getExceptionEvents());
    Assert.assertEquals(diagnosticsStreamMessage.getProcessorStopEvents(), Arrays.asList(new ProcessorStopEvent("0", EXECUTION_ENV_CONTAINER_ID, HOSTNAME, 102)));
    Assert.assertNull(diagnosticsStreamMessage.getContainerModels());
    Assert.assertNull(diagnosticsStreamMessage.getContainerNumCores());
    Assert.assertNull(diagnosticsStreamMessage.getNumPersistentStores());
}

Example 4 with MetricsSnapshotSerdeV2

use of org.apache.samza.serializers.MetricsSnapshotSerdeV2 in project samza by apache.

the class TestDiagnosticsManager method validateOutgoingMessageEnvelope.

private void validateOutgoingMessageEnvelope(OutgoingMessageEnvelope outgoingMessageEnvelope) {
    MetricsSnapshot metricsSnapshot = new MetricsSnapshotSerdeV2().fromBytes((byte[]) outgoingMessageEnvelope.getMessage());
    // Validate the diagnostics stream message
    DiagnosticsStreamMessage diagnosticsStreamMessage = DiagnosticsStreamMessage.convertToDiagnosticsStreamMessage(metricsSnapshot);
    Assert.assertEquals(CONTAINER_MB, diagnosticsStreamMessage.getContainerMb().intValue());
    Assert.assertEquals(MAX_HEAP_SIZE, diagnosticsStreamMessage.getMaxHeapSize().longValue());
    Assert.assertEquals(CONTAINER_THREAD_POOL_SIZE, diagnosticsStreamMessage.getContainerThreadPoolSize().intValue());
    Assert.assertEquals(exceptionEventList, diagnosticsStreamMessage.getExceptionEvents());
    Assert.assertEquals(diagnosticsStreamMessage.getProcessorStopEvents(), Arrays.asList(new ProcessorStopEvent("0", EXECUTION_ENV_CONTAINER_ID, HOSTNAME, 101)));
    Assert.assertEquals(containerModels, diagnosticsStreamMessage.getContainerModels());
    Assert.assertEquals(CONTAINER_NUM_CORES, diagnosticsStreamMessage.getContainerNumCores().intValue());
    Assert.assertEquals(NUM_PERSISTENT_STORES, diagnosticsStreamMessage.getNumPersistentStores().intValue());
    Assert.assertEquals(AUTOSIZING_ENABLED, diagnosticsStreamMessage.getAutosizingEnabled());
    Assert.assertEquals(config, diagnosticsStreamMessage.getConfig());
}

Example 5 with MetricsSnapshotSerdeV2

use of org.apache.samza.serializers.MetricsSnapshotSerdeV2 in project samza by apache.

the class TestDiagnosticsManager method testSecondPublishWithExceptionInSecondMessage.

@Test
public void testSecondPublishWithExceptionInSecondMessage() {
    // Across two successive run() invocations two messages should be published if stop events are added
    this.diagnosticsManager.start();
    DiagnosticsExceptionEvent diagnosticsExceptionEvent = new DiagnosticsExceptionEvent(System.currentTimeMillis(), new RuntimeException("exception"), new HashMap());
    this.diagnosticsManager.addExceptionEvent(diagnosticsExceptionEvent);
    this.diagnosticsManager.start();
    Assert.assertEquals("Two messages should have been published", 2, mockSystemProducer.getEnvelopeList().size());
    // Validate the first message
    OutgoingMessageEnvelope outgoingMessageEnvelope = mockSystemProducer.getEnvelopeList().get(0);
    validateMetricsHeader(outgoingMessageEnvelope, FIRST_SEND_TIME);
    validateOutgoingMessageEnvelope(outgoingMessageEnvelope);
    // Validate the second message's header
    outgoingMessageEnvelope = mockSystemProducer.getEnvelopeList().get(1);
    validateMetricsHeader(outgoingMessageEnvelope, SECOND_SEND_TIME);
    // Validate the second message's body (should be all empty except for the processor-stop-event)
    MetricsSnapshot metricsSnapshot = new MetricsSnapshotSerdeV2().fromBytes((byte[]) outgoingMessageEnvelope.getMessage());
    DiagnosticsStreamMessage diagnosticsStreamMessage = DiagnosticsStreamMessage.convertToDiagnosticsStreamMessage(metricsSnapshot);
    Assert.assertNull(diagnosticsStreamMessage.getContainerMb());
    Assert.assertEquals(Arrays.asList(diagnosticsExceptionEvent), diagnosticsStreamMessage.getExceptionEvents());
    Assert.assertNull(diagnosticsStreamMessage.getProcessorStopEvents());
    Assert.assertNull(diagnosticsStreamMessage.getContainerModels());
    Assert.assertNull(diagnosticsStreamMessage.getContainerNumCores());
    Assert.assertNull(diagnosticsStreamMessage.getNumPersistentStores());
}