Examples with CoordinatorStreamValueSerde org.apache.samza.coordinator.stream.CoordinatorStreamValueSerde used on opensource projects

Example 1 with CoordinatorStreamValueSerde

use of org.apache.samza.coordinator.stream.CoordinatorStreamValueSerde in project samza by apache.

the class TestContainerHeartbeatMonitor method testFailedToFetchNewAMCoordinatorUrl.

@Test
public void testFailedToFetchNewAMCoordinatorUrl() throws InterruptedException {
    this.containerHeartbeatMonitor = spy(buildContainerHeartbeatMonitor(true));
    CoordinatorStreamValueSerde serde = new CoordinatorStreamValueSerde(SetConfig.TYPE);
    ContainerHeartbeatMonitor.ContainerHeartbeatMetrics metrics = this.containerHeartbeatMonitor.getMetrics();
    when(this.containerHeartbeatClient.requestHeartbeat()).thenReturn(FAILURE_RESPONSE);
    when(this.coordinatorStreamStore.get(CoordinationConstants.YARN_COORDINATOR_URL)).thenReturn(serde.toBytes(COORDINATOR_URL));
    this.containerHeartbeatMonitor.start();
    // wait for the executor to finish the heartbeat check task
    boolean fixedRateTaskCompleted = this.schedulerFixedRateExecutionLatch.await(2, TimeUnit.SECONDS);
    assertTrue("Did not complete heartbeat check", fixedRateTaskCompleted);
    assertEquals("Heartbeat expired count should be 1", 1, metrics.getHeartbeatExpiredCount().getCount());
    assertEquals("Heartbeat established failure count should be 1", 1, metrics.getHeartbeatEstablishedFailureCount().getCount());
    // shutdown task should have been submitted
    verify(this.scheduler).schedule(any(Runnable.class), eq((long) ContainerHeartbeatMonitor.SHUTDOWN_TIMOUT_MS), eq(TimeUnit.MILLISECONDS));
    verify(this.onExpired).run();
    this.containerHeartbeatMonitor.stop();
    verify(this.scheduler).shutdown();
}

Example 2 with CoordinatorStreamValueSerde

use of org.apache.samza.coordinator.stream.CoordinatorStreamValueSerde in project samza by apache.

the class ZkJobCoordinator method loadMetadataResources.

/**
 * Stores the configuration of the job in the coordinator stream.
 */
@VisibleForTesting
void loadMetadataResources(JobModel jobModel) {
    try {
        MetadataResourceUtil metadataResourceUtil = createMetadataResourceUtil(jobModel, config);
        metadataResourceUtil.createResources();
        if (coordinatorStreamStore != null) {
            // TODO: SAMZA-2273 - publish configs async
            CoordinatorStreamValueSerde jsonSerde = new CoordinatorStreamValueSerde(SetConfig.TYPE);
            NamespaceAwareCoordinatorStreamStore configStore = new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE);
            for (Map.Entry<String, String> entry : config.entrySet()) {
                byte[] serializedValue = jsonSerde.toBytes(entry.getValue());
                configStore.put(entry.getKey(), serializedValue);
            }
            configStore.flush();
            if (new JobConfig(config).getStartpointEnabled()) {
                // fan out the startpoints
                StartpointManager startpointManager = createStartpointManager();
                startpointManager.start();
                try {
                    startpointManager.fanOut(JobModelUtil.getTaskToSystemStreamPartitions(jobModel));
                } finally {
                    startpointManager.stop();
                }
            }
        } else {
            LOG.warn("No metadata store registered to this job coordinator. Config not written to the metadata store and no Startpoints fan out.");
        }
    } catch (IOException ex) {
        throw new SamzaException(String.format("IO exception while loading metadata resources."), ex);
    }
}

Example 3 with CoordinatorStreamValueSerde

use of org.apache.samza.coordinator.stream.CoordinatorStreamValueSerde in project samza by apache.

the class TestJobCoordinatorMetadataManager method testReadJobCoordinatorMetadataFailed.

@Test
public void testReadJobCoordinatorMetadataFailed() {
    JobCoordinatorMetadata jobCoordinatorMetadata = new JobCoordinatorMetadata(NEW_EPOCH_ID, NEW_CONFIG_ID, NEW_JOB_MODEL_ID);
    Serde<String> mockSerde = spy(new CoordinatorStreamValueSerde(SetJobCoordinatorMetadataMessage.TYPE));
    doThrow(new RuntimeException("Failed to read coordinator stream")).when(mockSerde).fromBytes(any());
    jobCoordinatorMetadataManager = spy(new JobCoordinatorMetadataManager(metadataStore, ClusterType.YARN, new MetricsRegistryMap(), mockSerde));
    jobCoordinatorMetadataManager.writeJobCoordinatorMetadata(jobCoordinatorMetadata);
    JobCoordinatorMetadata actualMetadata = jobCoordinatorMetadataManager.readJobCoordinatorMetadata();
    assertNull("Read failed should return null", actualMetadata);
    assertEquals("Metadata read failed count should be 1", 1, jobCoordinatorMetadataManager.getMetrics().getMetadataReadFailedCount().getValue().intValue());
}

Example 4 with CoordinatorStreamValueSerde

use of org.apache.samza.coordinator.stream.CoordinatorStreamValueSerde in project samza by apache.

the class TestZkLocalApplicationRunner method getConfigFromCoordinatorStream.

private MapConfig getConfigFromCoordinatorStream(Config config) {
    MetadataStoreFactory metadataStoreFactory = ReflectionUtil.getObj(new JobConfig(config).getMetadataStoreFactory(), MetadataStoreFactory.class);
    MetadataStore metadataStore = metadataStoreFactory.getMetadataStore("set-config", config, new MetricsRegistryMap());
    metadataStore.init();
    Map<String, String> configMap = new HashMap<>();
    CoordinatorStreamValueSerde jsonSerde = new CoordinatorStreamValueSerde("set-config");
    metadataStore.all().forEach((key, value) -> {
        CoordinatorStreamStore.CoordinatorMessageKey coordinatorMessageKey = CoordinatorStreamStore.deserializeCoordinatorMessageKeyFromJson(key);
        String deserializedValue = jsonSerde.fromBytes(value);
        configMap.put(coordinatorMessageKey.getKey(), deserializedValue);
    });
    return new MapConfig(configMap);
}

Example 5 with CoordinatorStreamValueSerde

use of org.apache.samza.coordinator.stream.CoordinatorStreamValueSerde in project samza by apache.

the class TestContainerHeartbeatMonitor method testReestablishConnectionWithNewAM.

@Test
public void testReestablishConnectionWithNewAM() throws InterruptedException {
    String newCoordinatorUrl = "http://some-host-2.prod.linkedin.com";
    this.containerHeartbeatMonitor = spy(buildContainerHeartbeatMonitor(true));
    CoordinatorStreamValueSerde serde = new CoordinatorStreamValueSerde(SetConfig.TYPE);
    ContainerHeartbeatMonitor.ContainerHeartbeatMetrics metrics = this.containerHeartbeatMonitor.getMetrics();
    when(this.containerHeartbeatClient.requestHeartbeat()).thenReturn(FAILURE_RESPONSE).thenReturn(SUCCESS_RESPONSE);
    when(this.containerHeartbeatMonitor.createContainerHeartbeatClient(newCoordinatorUrl, CONTAINER_EXECUTION_ID)).thenReturn(this.containerHeartbeatClient);
    when(this.coordinatorStreamStore.get(CoordinationConstants.YARN_COORDINATOR_URL)).thenReturn(serde.toBytes(newCoordinatorUrl));
    this.containerHeartbeatMonitor.start();
    // wait for the executor to finish the heartbeat check task
    boolean fixedRateTaskCompleted = this.schedulerFixedRateExecutionLatch.await(2, TimeUnit.SECONDS);
    assertTrue("Did not complete heartbeat check", fixedRateTaskCompleted);
    assertEquals("Heartbeat expired count should be 1", 1, metrics.getHeartbeatExpiredCount().getCount());
    assertEquals("Heartbeat established failure count should be 0", 0, metrics.getHeartbeatEstablishedFailureCount().getCount());
    assertEquals("Heartbeat established with new AM should be 1", 1, metrics.getHeartbeatEstablishedWithNewAmCount().getCount());
    // shutdown task should not have been submitted
    verify(this.scheduler, never()).schedule(any(Runnable.class), anyLong(), any());
    verify(this.onExpired, never()).run();
    this.containerHeartbeatMonitor.stop();
    verify(this.scheduler).shutdown();
}