Examples with RetryStrategy io.cdap.cdap.common.service.RetryStrategy used on opensource projects

Example 6 with RetryStrategy

use of io.cdap.cdap.common.service.RetryStrategy in project cdap by caskdata.

the class SystemArgumentsTest method testRetryStrategies.

@Test
public void testRetryStrategies() {
    CConfiguration cConf = CConfiguration.create();
    Map<String, String> args = Collections.emptyMap();
    // Get default, expect exponential back-off behavior, until the max delay
    RetryStrategy strategy = SystemArguments.getRetryStrategy(args, ProgramType.CUSTOM_ACTION, cConf);
    long startTime = System.currentTimeMillis();
    Assert.assertEquals(1000L, strategy.nextRetry(1, startTime));
    Assert.assertEquals(2000L, strategy.nextRetry(2, startTime));
    Assert.assertEquals(4000L, strategy.nextRetry(3, startTime));
    Assert.assertEquals(8000L, strategy.nextRetry(4, startTime));
    Assert.assertEquals(16000L, strategy.nextRetry(5, startTime));
    Assert.assertEquals(30000L, strategy.nextRetry(6, startTime));
    Assert.assertEquals(30000L, strategy.nextRetry(7, startTime));
    // It should give up (returning -1) when exceeding the max retries
    Assert.assertEquals(-1L, strategy.nextRetry(1001, startTime));
    // Override the strategy type and max retry time
    args = ImmutableMap.of("system." + Constants.Retry.TYPE, RetryStrategyType.FIXED_DELAY.toString(), "system." + Constants.Retry.MAX_TIME_SECS, "5");
    strategy = SystemArguments.getRetryStrategy(args, ProgramType.CUSTOM_ACTION, cConf);
    startTime = System.currentTimeMillis();
    // Expects the delay doesn't change
    Assert.assertEquals(1000L, strategy.nextRetry(1, startTime));
    Assert.assertEquals(1000L, strategy.nextRetry(2, startTime));
    Assert.assertEquals(1000L, strategy.nextRetry(3, startTime));
    Assert.assertEquals(1000L, strategy.nextRetry(4, startTime));
    // Should give up (returning -1) after passing the max retry time
    Assert.assertEquals(-1L, strategy.nextRetry(1, startTime - 6000));
}

Example 7 with RetryStrategy

use of io.cdap.cdap.common.service.RetryStrategy in project cdap by caskdata.

the class MessagingUsageWriter method doRegisterAll.

private void doRegisterAll(Iterable<? extends EntityId> users, EntityId entityId) throws Exception {
    // Only record usage from program
    StoreRequest request = StoreRequestBuilder.of(topic).addPayloads(StreamSupport.stream(users.spliterator(), false).filter(ProgramId.class::isInstance).map(ProgramId.class::cast).map(id -> new MetadataMessage(MetadataMessage.Type.USAGE, id, GSON.toJsonTree(new DatasetUsage(entityId)))).map(GSON::toJson).map(s -> s.getBytes(StandardCharsets.UTF_8)).iterator()).build();
    Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
}

Example 8 with RetryStrategy

use of io.cdap.cdap.common.service.RetryStrategy in project cdap by cdapio.

the class SystemArgumentsTest method testRetryStrategies.

@Test
public void testRetryStrategies() {
    CConfiguration cConf = CConfiguration.create();
    Map<String, String> args = Collections.emptyMap();
    // Get default, expect exponential back-off behavior, until the max delay
    RetryStrategy strategy = SystemArguments.getRetryStrategy(args, ProgramType.CUSTOM_ACTION, cConf);
    long startTime = System.currentTimeMillis();
    Assert.assertEquals(1000L, strategy.nextRetry(1, startTime));
    Assert.assertEquals(2000L, strategy.nextRetry(2, startTime));
    Assert.assertEquals(4000L, strategy.nextRetry(3, startTime));
    Assert.assertEquals(8000L, strategy.nextRetry(4, startTime));
    Assert.assertEquals(16000L, strategy.nextRetry(5, startTime));
    Assert.assertEquals(30000L, strategy.nextRetry(6, startTime));
    Assert.assertEquals(30000L, strategy.nextRetry(7, startTime));
    // It should give up (returning -1) when exceeding the max retries
    Assert.assertEquals(-1L, strategy.nextRetry(1001, startTime));
    // Override the strategy type and max retry time
    args = ImmutableMap.of("system." + Constants.Retry.TYPE, RetryStrategyType.FIXED_DELAY.toString(), "system." + Constants.Retry.MAX_TIME_SECS, "5");
    strategy = SystemArguments.getRetryStrategy(args, ProgramType.CUSTOM_ACTION, cConf);
    startTime = System.currentTimeMillis();
    // Expects the delay doesn't change
    Assert.assertEquals(1000L, strategy.nextRetry(1, startTime));
    Assert.assertEquals(1000L, strategy.nextRetry(2, startTime));
    Assert.assertEquals(1000L, strategy.nextRetry(3, startTime));
    Assert.assertEquals(1000L, strategy.nextRetry(4, startTime));
    // Should give up (returning -1) after passing the max retry time
    Assert.assertEquals(-1L, strategy.nextRetry(1, startTime - 6000));
}

Example 9 with RetryStrategy

use of io.cdap.cdap.common.service.RetryStrategy in project cdap by cdapio.

the class RuntimeServiceMainTest method createProgramStateWriter.

/**
 * Creates a {@link ProgramStateWriter} that writes to {@link RuntimeClient} directly.
 *
 * @param injector the injector for creating the {@link RuntimeClient}
 * @param programRunId the {@link ProgramRunId} for the program state change
 * @return a {@link ProgramStateWriter}
 */
private ProgramStateWriter createProgramStateWriter(Injector injector, ProgramRunId programRunId) {
    RuntimeClient runtimeClient = injector.getInstance(RuntimeClient.class);
    // We write to the record event directly to skip the app-fabric to process it
    // This is because we don't follow the normal event flow here for testing
    TopicId topicId = NamespaceId.SYSTEM.topic(injector.getInstance(CConfiguration.class).get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
    RetryStrategy retryStrategy = RetryStrategies.timeLimit(5, TimeUnit.SECONDS, RetryStrategies.fixDelay(200, TimeUnit.MILLISECONDS));
    return new MessagingProgramStateWriter((notificationType, properties) -> {
        Notification notification = new Notification(notificationType, properties);
        try {
            Retries.callWithRetries((Retries.Callable<Void, Exception>) () -> {
                runtimeClient.sendMessages(programRunId, topicId, Collections.singleton(createMessage(notification)).iterator());
                return null;
            }, retryStrategy, t -> t instanceof IOException || t instanceof RetryableException);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
}

Example 10 with RetryStrategy

use of io.cdap.cdap.common.service.RetryStrategy in project cdap by cdapio.

the class DistributedLogFramework method createService.

@Override
@SuppressWarnings("unchecked")
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix() + topic, CheckpointManagerFactory.Type.KAFKA), brokerService, config), retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::start));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::stop));
        }
    };
}