use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.
the class TestLocalityServlet method testReadProcessorLocalityWithNoLocality.
@Test
public void testReadProcessorLocalityWithNoLocality() throws Exception {
final ProcessorLocality expectedProcessorLocality = new ProcessorLocality(PROCESSOR_ID2, "");
URL url = new URL(webApp.getUrl().toString() + "locality?processorId=" + PROCESSOR_ID2);
when(localityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of()));
String response = HttpUtil.read(url, 1000, new ExponentialSleepStrategy());
ProcessorLocality processorLocality = mapper.readValue(response, ProcessorLocality.class);
assertEquals("Expected empty response for processor locality " + PROCESSOR_ID2 + " but got " + processorLocality, processorLocality, expectedProcessorLocality);
}
use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.
the class KafkaSystemAdmin method getSSPMetadata.
/**
* Given a set of SystemStreamPartition, fetch metadata from Kafka for each
* of them, and return a map from ssp to SystemStreamPartitionMetadata for
* each of them. This method will return null for oldest and newest offsets
* if a given SystemStreamPartition is empty. This method will block and
* retry indefinitely until it gets a successful response from Kafka.
* @param ssps a set of strings of SSP
* @param retryBackoff retry backoff strategy
* @return a map from ssp to sspMetadata which has offsets
*/
Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> getSSPMetadata(Set<SystemStreamPartition> ssps, ExponentialSleepStrategy retryBackoff) {
LOG.info("Fetching SSP metadata for: {}", ssps);
List<TopicPartition> topicPartitions = ssps.stream().map(ssp -> new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId())).collect(Collectors.toList());
Function1<ExponentialSleepStrategy.RetryLoop, Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata>> fetchTopicPartitionMetadataOperation = new AbstractFunction1<ExponentialSleepStrategy.RetryLoop, Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata>>() {
@Override
public Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> apply(ExponentialSleepStrategy.RetryLoop loop) {
OffsetsMaps topicPartitionsMetadata = fetchTopicPartitionsMetadata(topicPartitions);
Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> sspToSSPMetadata = new HashMap<>();
for (SystemStreamPartition ssp : ssps) {
String oldestOffset = topicPartitionsMetadata.getOldestOffsets().get(ssp);
String newestOffset = topicPartitionsMetadata.getNewestOffsets().get(ssp);
String upcomingOffset = topicPartitionsMetadata.getUpcomingOffsets().get(ssp);
sspToSSPMetadata.put(ssp, new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset));
}
loop.done();
return sspToSSPMetadata;
}
};
Function2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit> onExceptionRetryOperation = new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() {
@Override
public BoxedUnit apply(Exception exception, ExponentialSleepStrategy.RetryLoop loop) {
if (loop.sleepCount() < MAX_RETRIES_ON_EXCEPTION) {
LOG.warn(String.format("Fetching SSP metadata for: %s threw an exception. Retrying.", ssps), exception);
} else {
LOG.error(String.format("Fetching SSP metadata for: %s threw an exception.", ssps), exception);
loop.done();
throw new SamzaException(exception);
}
return null;
}
};
Function0<Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata>> fallbackOperation = new AbstractFunction0<Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata>>() {
@Override
public Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> apply() {
throw new SamzaException("Failed to get SSP metadata");
}
};
return retryBackoff.run(fetchTopicPartitionMetadataOperation, onExceptionRetryOperation).getOrElse(fallbackOperation);
}
use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.
the class TestKafkaSystemAdminWithMock method testGetSSPMetadataWithRetry.
@Test
public void testGetSSPMetadataWithRetry() {
SystemStreamPartition oneSSP = new SystemStreamPartition(TEST_SYSTEM, VALID_TOPIC, new Partition(0));
SystemStreamPartition otherSSP = new SystemStreamPartition(TEST_SYSTEM, "otherTopic", new Partition(1));
ImmutableSet<SystemStreamPartition> ssps = ImmutableSet.of(oneSSP, otherSSP);
List<TopicPartition> topicPartitions = ssps.stream().map(ssp -> new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId())).collect(Collectors.toList());
Map<TopicPartition, Long> testBeginningOffsets = ImmutableMap.of(testTopicPartition0, KAFKA_BEGINNING_OFFSET_FOR_PARTITION0, testTopicPartition1, KAFKA_BEGINNING_OFFSET_FOR_PARTITION1);
when(mockKafkaConsumer.beginningOffsets(topicPartitions)).thenThrow(new RuntimeException()).thenReturn(testBeginningOffsets);
Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> sspMetadata = kafkaSystemAdmin.getSSPMetadata(ssps, new ExponentialSleepStrategy(2, 1, 1));
assertEquals("metadata should return for 2 topics", sspMetadata.size(), 2);
// retried twice because the first fails and the second succeeds
Mockito.verify(mockKafkaConsumer, Mockito.times(2)).beginningOffsets(topicPartitions);
}
use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.
the class TestKafkaSystemAdminWithMock method testGetSSPMetadataShouldTerminateAfterFiniteRetriesOnException.
@Test(expected = SamzaException.class)
public void testGetSSPMetadataShouldTerminateAfterFiniteRetriesOnException() throws Exception {
SystemStreamPartition oneSSP = new SystemStreamPartition(TEST_SYSTEM, VALID_TOPIC, new Partition(0));
SystemStreamPartition otherSSP = new SystemStreamPartition(TEST_SYSTEM, "otherTopic", new Partition(1));
ImmutableSet<SystemStreamPartition> ssps = ImmutableSet.of(oneSSP, otherSSP);
List<TopicPartition> topicPartitions = ssps.stream().map(ssp -> new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId())).collect(Collectors.toList());
when(mockKafkaConsumer.beginningOffsets(topicPartitions)).thenThrow(new RuntimeException()).thenThrow(new RuntimeException());
kafkaSystemAdmin.getSSPMetadata(ssps, new ExponentialSleepStrategy(2, 1, 1));
}
use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.
the class TestLocalityServlet method testReadContainerLocalityWithNoLocality.
@Test
public void testReadContainerLocalityWithNoLocality() throws Exception {
final LocalityModel expectedLocality = new LocalityModel(Collections.emptyMap());
URL url = new URL(webApp.getUrl().toString() + "locality");
when(localityManager.readLocality()).thenReturn(new LocalityModel(ImmutableMap.of()));
String response = HttpUtil.read(url, 1000, new ExponentialSleepStrategy());
LocalityModel locality = mapper.readValue(response, LocalityModel.class);
assertEquals("Expected empty response but got " + locality, locality, expectedLocality);
}
Aggregations