Examples with AmazonKinesisClient com.amazonaws.services.kinesis.AmazonKinesisClient used on opensource projects

Example 1 with AmazonKinesisClient

use of com.amazonaws.services.kinesis.AmazonKinesisClient in project flink by apache.

the class ManualExactlyOnceTest method main.

public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    LOG.info("Starting exactly once test");
    final String streamName = "flink-test-" + UUID.randomUUID().toString();
    final String accessKey = pt.getRequired("accessKey");
    final String secretKey = pt.getRequired("secretKey");
    final String region = pt.getRequired("region");
    Properties configProps = new Properties();
    configProps.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
    configProps.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
    configProps.setProperty(AWSConfigConstants.AWS_REGION, region);
    AmazonKinesisClient client = AWSUtil.createKinesisClient(configProps);
    // create a stream for the test:
    client.createStream(streamName, 1);
    // wait until stream has been created
    DescribeStreamResult status = client.describeStream(streamName);
    LOG.info("status {}", status);
    while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
        status = client.describeStream(streamName);
        LOG.info("Status of stream {}", status);
        Thread.sleep(1000);
    }
    final Configuration flinkConfig = new Configuration();
    flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 16);
    flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
    LocalFlinkMiniCluster flink = new LocalFlinkMiniCluster(flinkConfig, false);
    flink.start();
    final int flinkPort = flink.getLeaderRPCPort();
    try {
        final AtomicReference<Throwable> producerError = new AtomicReference<>();
        Thread producerThread = KinesisEventsGeneratorProducerThread.create(TOTAL_EVENT_COUNT, 2, accessKey, secretKey, region, streamName, producerError, flinkPort, flinkConfig);
        producerThread.start();
        final AtomicReference<Throwable> consumerError = new AtomicReference<>();
        Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(TOTAL_EVENT_COUNT, 200, 2, 500, 500, accessKey, secretKey, region, streamName, consumerError, flinkPort, flinkConfig);
        consumerThread.start();
        boolean deadlinePassed = false;
        // wait at most for two minutes
        long deadline = System.currentTimeMillis() + (1000 * 2 * 60);
        // wait until both producer and consumer finishes, or an unexpected error is thrown
        while ((consumerThread.isAlive() || producerThread.isAlive()) && (producerError.get() == null && consumerError.get() == null)) {
            Thread.sleep(1000);
            if (System.currentTimeMillis() >= deadline) {
                LOG.warn("Deadline passed");
                deadlinePassed = true;
                // enough waiting
                break;
            }
        }
        if (producerThread.isAlive()) {
            producerThread.interrupt();
        }
        if (consumerThread.isAlive()) {
            consumerThread.interrupt();
        }
        if (producerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Producer failed", producerError.get());
        }
        if (consumerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Consumer failed", consumerError.get());
        }
        if (!deadlinePassed) {
            LOG.info("+++ TEST passed! +++");
        } else {
            LOG.info("+++ TEST failed! +++");
        }
    } finally {
        client.deleteStream(streamName);
        client.shutdown();
        // stopping flink
        flink.stop();
    }
}

Example 2 with AmazonKinesisClient

use of com.amazonaws.services.kinesis.AmazonKinesisClient in project flink by apache.

the class AWSUtil method createKinesisClient.

/**
	 * Creates an Amazon Kinesis Client.
	 * @param configProps configuration properties containing the access key, secret key, and region
	 * @return a new Amazon Kinesis Client
	 */
public static AmazonKinesisClient createKinesisClient(Properties configProps) {
    // set a Flink-specific user agent
    ClientConfiguration awsClientConfig = new ClientConfigurationFactory().getConfig();
    awsClientConfig.setUserAgent("Apache Flink " + EnvironmentInformation.getVersion() + " (" + EnvironmentInformation.getRevisionInformation().commitId + ") Kinesis Connector");
    // utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
    AmazonKinesisClient client = new AmazonKinesisClient(AWSUtil.getCredentialsProvider(configProps), awsClientConfig);
    client.setRegion(Region.getRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION))));
    if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
        client.setEndpoint(configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT));
    }
    return client;
}

Example 3 with AmazonKinesisClient

use of com.amazonaws.services.kinesis.AmazonKinesisClient in project flink by apache.

the class ManualExactlyOnceWithStreamReshardingTest method main.

public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    LOG.info("Starting exactly once with stream resharding test");
    final String streamName = "flink-test-" + UUID.randomUUID().toString();
    final String accessKey = pt.getRequired("accessKey");
    final String secretKey = pt.getRequired("secretKey");
    final String region = pt.getRequired("region");
    final Properties configProps = new Properties();
    configProps.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
    configProps.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
    configProps.setProperty(ConsumerConfigConstants.AWS_REGION, region);
    configProps.setProperty(ConsumerConfigConstants.SHARD_DISCOVERY_INTERVAL_MILLIS, "0");
    final AmazonKinesisClient client = AWSUtil.createKinesisClient(configProps);
    // the stream is first created with 1 shard
    client.createStream(streamName, 1);
    // wait until stream has been created
    DescribeStreamResult status = client.describeStream(streamName);
    LOG.info("status {}", status);
    while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
        status = client.describeStream(streamName);
        LOG.info("Status of stream {}", status);
        Thread.sleep(1000);
    }
    final Configuration flinkConfig = new Configuration();
    flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 16);
    flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
    LocalFlinkMiniCluster flink = new LocalFlinkMiniCluster(flinkConfig, false);
    flink.start();
    final int flinkPort = flink.getLeaderRPCPort();
    try {
        // we have to use a manual generator here instead of the FlinkKinesisProducer
        // because the FlinkKinesisProducer currently has a problem where records will be resent to a shard
        // when resharding happens; this affects the consumer exactly-once validation test and will never pass
        final AtomicReference<Throwable> producerError = new AtomicReference<>();
        Runnable manualGenerate = new Runnable() {

            @Override
            public void run() {
                AmazonKinesisClient client = AWSUtil.createKinesisClient(configProps);
                int count = 0;
                final int batchSize = 30;
                while (true) {
                    try {
                        Thread.sleep(10);
                        Set<PutRecordsRequestEntry> batch = new HashSet<>();
                        for (int i = count; i < count + batchSize; i++) {
                            if (i >= TOTAL_EVENT_COUNT) {
                                break;
                            }
                            batch.add(new PutRecordsRequestEntry().withData(ByteBuffer.wrap(((i) + "-" + RandomStringUtils.randomAlphabetic(12)).getBytes(ConfigConstants.DEFAULT_CHARSET))).withPartitionKey(UUID.randomUUID().toString()));
                        }
                        count += batchSize;
                        PutRecordsResult result = client.putRecords(new PutRecordsRequest().withStreamName(streamName).withRecords(batch));
                        // and let this test fail
                        if (result.getFailedRecordCount() > 0) {
                            producerError.set(new RuntimeException("The producer has failed records in one of the put batch attempts."));
                            break;
                        }
                        if (count >= TOTAL_EVENT_COUNT) {
                            break;
                        }
                    } catch (Exception e) {
                        producerError.set(e);
                    }
                }
            }
        };
        Thread producerThread = new Thread(manualGenerate);
        producerThread.start();
        final AtomicReference<Throwable> consumerError = new AtomicReference<>();
        Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(TOTAL_EVENT_COUNT, 10000, 2, 500, 500, accessKey, secretKey, region, streamName, consumerError, flinkPort, flinkConfig);
        consumerThread.start();
        // reshard the Kinesis stream while the producer / and consumers are running
        Runnable splitShard = new Runnable() {

            @Override
            public void run() {
                try {
                    // first, split shard in the middle of the hash range
                    Thread.sleep(5000);
                    LOG.info("Splitting shard ...");
                    client.splitShard(streamName, KinesisShardIdGenerator.generateFromShardOrder(0), "170141183460469231731687303715884105727");
                    // wait until the split shard operation finishes updating ...
                    DescribeStreamResult status;
                    Random rand = new Random();
                    do {
                        status = null;
                        while (status == null) {
                            // retry until we get status
                            try {
                                status = client.describeStream(streamName);
                            } catch (LimitExceededException lee) {
                                LOG.warn("LimitExceededException while describing stream ... retrying ...");
                                Thread.sleep(rand.nextInt(1200));
                            }
                        }
                    } while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE"));
                    // then merge again
                    Thread.sleep(7000);
                    LOG.info("Merging shards ...");
                    client.mergeShards(streamName, KinesisShardIdGenerator.generateFromShardOrder(1), KinesisShardIdGenerator.generateFromShardOrder(2));
                } catch (InterruptedException iex) {
                //
                }
            }
        };
        Thread splitShardThread = new Thread(splitShard);
        splitShardThread.start();
        boolean deadlinePassed = false;
        // wait at most for five minutes
        long deadline = System.currentTimeMillis() + (1000 * 5 * 60);
        // wait until both producer and consumer finishes, or an unexpected error is thrown
        while ((consumerThread.isAlive() || producerThread.isAlive()) && (producerError.get() == null && consumerError.get() == null)) {
            Thread.sleep(1000);
            if (System.currentTimeMillis() >= deadline) {
                LOG.warn("Deadline passed");
                deadlinePassed = true;
                // enough waiting
                break;
            }
        }
        if (producerThread.isAlive()) {
            producerThread.interrupt();
        }
        if (consumerThread.isAlive()) {
            consumerThread.interrupt();
        }
        if (producerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Producer failed", producerError.get());
        }
        if (consumerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Consumer failed", consumerError.get());
        }
        if (!deadlinePassed) {
            LOG.info("+++ TEST passed! +++");
        } else {
            LOG.info("+++ TEST failed! +++");
        }
    } finally {
        client.deleteStream(streamName);
        client.shutdown();
        // stopping flink
        flink.stop();
    }
}

Example 4 with AmazonKinesisClient

use of com.amazonaws.services.kinesis.AmazonKinesisClient in project storm by apache.

the class KinesisConnection method initialize.

void initialize() {
    kinesisClient = new AmazonKinesisClient(kinesisConnectionInfo.getCredentialsProvider(), kinesisConnectionInfo.getClientConfiguration());
    kinesisClient.setRegion(Region.getRegion(kinesisConnectionInfo.getRegion()));
}

Example 5 with AmazonKinesisClient

use of com.amazonaws.services.kinesis.AmazonKinesisClient in project voltdb by VoltDB.

the class KinesisStreamImporterConfig method discoverShards.

/**
     * connect to kinesis stream to discover the shards on the stream
     *
     * @param regionName The region name where the stream resides
     * @param streamName The kinesis stream name
     * @param accessKey The user access key
     * @param secretKey The user secret key
     * @param appName  The name of stream application
     * @return a list of shards
     */
public static List<Shard> discoverShards(String regionName, String streamName, String accessKey, String secretKey, String appName) {
    try {
        Region region = RegionUtils.getRegion(regionName);
        if (region != null) {
            final AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
            AmazonKinesis kinesisClient = new AmazonKinesisClient(credentials, getClientConfigWithUserAgent(appName));
            kinesisClient.setRegion(region);
            DescribeStreamResult result = kinesisClient.describeStream(streamName);
            if (!"ACTIVE".equals(result.getStreamDescription().getStreamStatus())) {
                throw new IllegalArgumentException("Kinesis stream " + streamName + " is not active.");
            }
            return result.getStreamDescription().getShards();
        }
    } catch (ResourceNotFoundException e) {
        LOGGER.warn("Kinesis stream " + streamName + " does not exist.", e);
    } catch (Exception e) {
        LOGGER.warn("Error found while describing the kinesis stream " + streamName, e);
    }
    return null;
}