Examples with MemLeaseManager com.demandware.carbonj.service.engine.kinesis.kcl.MemLeaseManager used on opensource projects

Example 1 with MemLeaseManager

use of com.demandware.carbonj.service.engine.kinesis.kcl.MemLeaseManager in project carbonj by salesforce.

the class KinesisConsumer method run.

public void run() {
    while (!closed) {
        try {
            AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
            String workerId = kinesisApplicationName + "-worker";
            if (kinesisConfig.isRecoveryEnabled()) {
                initCatchupKinesisClient();
            }
            KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(kinesisApplicationName, kinesisStreamName, credentialsProvider, workerId).withInitialPositionInStream(InitialPositionInStream.LATEST).withFailoverTimeMillis(kinesisConfig.getLeaseExpirationTimeInSecs() * 1000).withRegionName(kinesisConsumerRegion);
            int maxRecords = kinesisConfig.getMaxRecords();
            if (maxRecords > 0) {
                kinesisClientLibConfiguration.withMaxRecords(maxRecords);
            }
            log.info(" Kinesis Client Library started with application name " + kinesisApplicationName + " with stream " + kinesisStreamName + " and worker id is " + workerId);
            IRecordProcessorFactory recordProcessorFactory = new KinesisRecordProcessorFactory(metricRegistry, pointProcessor, kinesisConfig, kinesisStreamName);
            worker = new Worker.Builder().recordProcessorFactory(recordProcessorFactory).config(kinesisClientLibConfiguration).leaseManager(new MemLeaseManager(kinesisConfig.getLeaseTakerDelayInMillis())).build();
            worker.run();
        } catch (Throwable t) {
            log.error("Error in initializing kinesis consumer", t);
            shutdownQuietly(worker);
            try {
                Thread.sleep(TimeUnit.SECONDS.toMillis(kinesisConfig.getInitRetryTimeInSecs()));
            } catch (InterruptedException e) {
                log.error("wait interrupted", e);
            }
            noOfRestarts.inc();
        }
    }
}

Example 2 with MemLeaseManager

use of com.demandware.carbonj.service.engine.kinesis.kcl.MemLeaseManager in project carbonj by salesforce.

the class TestMemLeaseManager method testBasicPath.

@Test
public void testBasicPath() throws Exception {
    ILeaseManager<KinesisClientLease> leaseManager = new MemLeaseManager<>(1);
    // check if the lease table exists
    Assert.assertFalse(leaseManager.leaseTableExists());
    // create table if it not exists
    Assert.assertTrue(leaseManager.createLeaseTableIfNotExists(10L, 10L));
    // check if the lease table exists
    Assert.assertTrue(leaseManager.waitUntilLeaseTableExists(1, 10));
    // check if the lease table exists
    Assert.assertTrue(leaseManager.leaseTableExists());
    // check if the lease table is empty
    Assert.assertTrue(leaseManager.isLeaseTableEmpty());
    Assert.assertEquals(0, leaseManager.listLeases().size());
    // create lease for 5 shards
    for (int i = 0; i < NO_OF_SHARDS; i++) {
        String shardId = "shard-" + i;
        KinesisClientLease lease = newKCLLease(shardId);
        Assert.assertTrue(leaseManager.createLeaseIfNotExists(lease));
    }
    // check if the lease table is not empty
    Assert.assertFalse(leaseManager.isLeaseTableEmpty());
    Assert.assertEquals(5, leaseManager.listLeases().size());
    verify(leaseManager, null, 0L);
    for (int i = 0; i < NO_OF_SHARDS; i++) {
        String shardId = "shard-" + i;
        KinesisClientLease lease = leaseManager.getLease(shardId);
        leaseManager.takeLease(lease, OWNER);
    }
    verify(leaseManager, OWNER, 1L);
    renewLeases(leaseManager);
    verify(leaseManager, OWNER, 2L);
    updateLeases(leaseManager);
    verify(leaseManager, OWNER, 3L);
}