Examples with ExponentialSleepStrategy org.apache.samza.util.ExponentialSleepStrategy used on opensource projects

Example 1 with ExponentialSleepStrategy

use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.

the class KafkaSystemAdmin method getSystemStreamPartitionCounts.

/**
 * Note! This method does not populate SystemStreamMetadata for each stream with real data.
 * Thus, this method should ONLY be used to get number of partitions for each stream.
 * It will throw NotImplementedException if anyone tries to access the actual metadata.
 * @param streamNames set of streams for which get the partitions counts
 * @param cacheTTL cache TTL if caching the data
 * @return a map, keyed on stream names. Number of partitions in SystemStreamMetadata is the output of this method.
 */
@Override
public Map<String, SystemStreamMetadata> getSystemStreamPartitionCounts(Set<String> streamNames, long cacheTTL) {
    // This optimization omits actual metadata for performance. Instead, we inject a dummy for all partitions.
    final SystemStreamMetadata.SystemStreamPartitionMetadata dummySspm = new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null) {

        String msg = "getSystemStreamPartitionCounts does not populate SystemStreaMetadata info. Only number of partitions";

        @Override
        public String getOldestOffset() {
            throw new NotImplementedException(msg);
        }

        @Override
        public String getNewestOffset() {
            throw new NotImplementedException(msg);
        }

        @Override
        public String getUpcomingOffset() {
            throw new NotImplementedException(msg);
        }
    };
    ExponentialSleepStrategy strategy = new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS);
    Function1<ExponentialSleepStrategy.RetryLoop, Map<String, SystemStreamMetadata>> fetchMetadataOperation = new AbstractFunction1<ExponentialSleepStrategy.RetryLoop, Map<String, SystemStreamMetadata>>() {

        @Override
        public Map<String, SystemStreamMetadata> apply(ExponentialSleepStrategy.RetryLoop loop) {
            Map<String, SystemStreamMetadata> allMetadata = new HashMap<>();
            streamNames.forEach(streamName -> {
                Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>();
                List<PartitionInfo> partitionInfos = threadSafeKafkaConsumer.execute(consumer -> consumer.partitionsFor(streamName));
                LOG.debug("Stream {} has partitions {}", streamName, partitionInfos);
                partitionInfos.forEach(partitionInfo -> partitionMetadata.put(new Partition(partitionInfo.partition()), dummySspm));
                allMetadata.put(streamName, new SystemStreamMetadata(streamName, partitionMetadata));
            });
            loop.done();
            return allMetadata;
        }
    };
    Map<String, SystemStreamMetadata> result = strategy.run(fetchMetadataOperation, new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() {

        @Override
        public BoxedUnit apply(Exception exception, ExponentialSleepStrategy.RetryLoop loop) {
            if (loop.sleepCount() < MAX_RETRIES_ON_EXCEPTION) {
                LOG.warn(String.format("Fetching systemstreampartition counts for: %s threw an exception. Retrying.", streamNames), exception);
            } else {
                LOG.error(String.format("Fetching systemstreampartition counts for: %s threw an exception.", streamNames), exception);
                loop.done();
                throw new SamzaException(exception);
            }
            return null;
        }
    }).get();
    LOG.info("SystemStream partition counts for system {}: {}", systemName, result);
    return result;
}

Example 2 with ExponentialSleepStrategy

use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.

the class TestKafkaSystemProducerJava method testInstantiateProducer.

@Test
public void testInstantiateProducer() {
    KafkaSystemProducer ksp = new KafkaSystemProducer("SysName", new ExponentialSleepStrategy(2.0, 200, 10000), new AbstractFunction0<Producer<byte[], byte[]>>() {

        @Override
        public Producer<byte[], byte[]> apply() {
            return new KafkaProducer<>(new HashMap<String, Object>());
        }
    }, new KafkaSystemProducerMetrics("SysName", new MetricsRegistryMap()), new AbstractFunction0<Object>() {

        @Override
        public Object apply() {
            return System.currentTimeMillis();
        }
    }, false);
    long now = System.currentTimeMillis();
    assertTrue((Long) ksp.clock().apply() >= now);
}

Example 3 with ExponentialSleepStrategy

use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.

the class TestYarnContainerHeartbeatServlet method testContainerHeartbeatWhenInvalid.

@Test
public void testContainerHeartbeatWhenInvalid() throws IOException {
    String validContainerId = "container_1350670447861_0003_01_000003";
    String invalidContainerId = "container_1350670447861_0003_01_000002";
    when(container.id()).thenReturn(ConverterUtils.toContainerId(validContainerId));
    yarnAppState.runningProcessors.put(validContainerId, container);
    URL url = new URL(String.format(CoordinationConstants.YARN_CONTAINER_HEARTBEAT_ENDPOINT_FORMAT, webApp.getUrl().toString(), invalidContainerId));
    String response = HttpUtil.read(url, 1000, new ExponentialSleepStrategy());
    heartbeat = mapper.readValue(response, ContainerHeartbeatResponse.class);
    Assert.assertFalse(heartbeat.isAlive());
}

Example 4 with ExponentialSleepStrategy

use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.

the class TestYarnContainerHeartbeatServlet method testContainerHeartbeatWhenValid.

@Test
public void testContainerHeartbeatWhenValid() throws IOException {
    String validContainerId = "container_1350670447861_0003_01_000002";
    when(container.id()).thenReturn(ConverterUtils.toContainerId(validContainerId));
    yarnAppState.runningProcessors.put(validContainerId, container);
    URL url = new URL(String.format(CoordinationConstants.YARN_CONTAINER_HEARTBEAT_ENDPOINT_FORMAT, webApp.getUrl().toString(), validContainerId));
    String response = HttpUtil.read(url, 1000, new ExponentialSleepStrategy());
    heartbeat = mapper.readValue(response, ContainerHeartbeatResponse.class);
    Assert.assertTrue(heartbeat.isAlive());
}

Example 5 with ExponentialSleepStrategy

use of org.apache.samza.util.ExponentialSleepStrategy in project samza by apache.

the class TestLocalityServlet method testReadContainerLocality.

@Test
public void testReadContainerLocality() throws Exception {
    URL url = new URL(webApp.getUrl().toString() + "locality");
    String response = HttpUtil.read(url, 1000, new ExponentialSleepStrategy());
    LocalityModel locality = mapper.readValue(response, LocalityModel.class);
    assertEquals("Expected locality for two containers", 2, locality.getProcessorLocalities().size());
    assertEquals("Mismatch in locality for processor " + PROCESSOR_ID1, locality.getProcessorLocality(PROCESSOR_ID1), PROCESSOR_1_LOCALITY);
    assertEquals("Mismatch in locality for processor " + PROCESSOR_ID2, locality.getProcessorLocality(PROCESSOR_ID2), PROCESSOR_2_LOCALITY);
}