Examples with AmazonKinesis com.amazonaws.services.kinesis.AmazonKinesis used on opensource projects

Example 1 with AmazonKinesis

use of com.amazonaws.services.kinesis.AmazonKinesis in project flink by apache.

the class KinesisProxyTest method testClientConfigOverride.

@Test
public void testClientConfigOverride() {
    Properties configProps = new Properties();
    configProps.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
    configProps.setProperty(AWSUtil.AWS_CLIENT_CONFIG_PREFIX + "socketTimeout", "9999");
    KinesisProxyInterface proxy = KinesisProxy.create(configProps);
    AmazonKinesis kinesisClient = Whitebox.getInternalState(proxy, "kinesisClient");
    ClientConfiguration clientConfiguration = Whitebox.getInternalState(kinesisClient, "clientConfiguration");
    assertEquals(9999, clientConfiguration.getSocketTimeout());
}

Example 2 with AmazonKinesis

use of com.amazonaws.services.kinesis.AmazonKinesis in project flink by apache.

the class KinesisProxyTest method testCustomConfigurationOverride.

@Test
public void testCustomConfigurationOverride() {
    Properties configProps = new Properties();
    configProps.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
    KinesisProxy proxy = new KinesisProxy(configProps) {

        @Override
        protected AmazonKinesis createKinesisClient(Properties configProps) {
            ClientConfiguration clientConfig = new ClientConfigurationFactory().getConfig();
            clientConfig.setSocketTimeout(10000);
            return AWSUtil.createKinesisClient(configProps, clientConfig);
        }
    };
    AmazonKinesis kinesisClient = Whitebox.getInternalState(proxy, "kinesisClient");
    ClientConfiguration clientConfiguration = Whitebox.getInternalState(kinesisClient, "clientConfiguration");
    assertEquals(10000, clientConfiguration.getSocketTimeout());
}

Example 3 with AmazonKinesis

use of com.amazonaws.services.kinesis.AmazonKinesis in project flink by apache.

the class ManualExactlyOnceTest method main.

public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    LOG.info("Starting exactly once test");
    final String streamName = "flink-test-" + UUID.randomUUID().toString();
    final String accessKey = pt.getRequired("accessKey");
    final String secretKey = pt.getRequired("secretKey");
    final String region = pt.getRequired("region");
    Properties configProps = new Properties();
    configProps.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
    configProps.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
    configProps.setProperty(AWSConfigConstants.AWS_REGION, region);
    AmazonKinesis client = AWSUtil.createKinesisClient(configProps);
    // create a stream for the test:
    client.createStream(streamName, 1);
    // wait until stream has been created
    DescribeStreamResult status = client.describeStream(streamName);
    LOG.info("status {}", status);
    while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
        status = client.describeStream(streamName);
        LOG.info("Status of stream {}", status);
        Thread.sleep(1000);
    }
    final Configuration flinkConfig = new Configuration();
    flinkConfig.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("16m"));
    flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
    MiniClusterResource flink = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder().setNumberTaskManagers(1).setNumberSlotsPerTaskManager(8).setConfiguration(flinkConfig).build());
    flink.before();
    final int flinkPort = flink.getRestAddres().getPort();
    try {
        final AtomicReference<Throwable> producerError = new AtomicReference<>();
        Thread producerThread = KinesisEventsGeneratorProducerThread.create(TOTAL_EVENT_COUNT, 2, accessKey, secretKey, region, streamName, producerError, flinkPort, flinkConfig);
        producerThread.start();
        final AtomicReference<Throwable> consumerError = new AtomicReference<>();
        Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(TOTAL_EVENT_COUNT, 200, 2, 500, 500, accessKey, secretKey, region, streamName, consumerError, flinkPort, flinkConfig);
        consumerThread.start();
        boolean deadlinePassed = false;
        long deadline = // wait at most for two minutes
        System.currentTimeMillis() + (1000 * 2 * 60);
        // wait until both producer and consumer finishes, or an unexpected error is thrown
        while ((consumerThread.isAlive() || producerThread.isAlive()) && (producerError.get() == null && consumerError.get() == null)) {
            Thread.sleep(1000);
            if (System.currentTimeMillis() >= deadline) {
                LOG.warn("Deadline passed");
                deadlinePassed = true;
                // enough waiting
                break;
            }
        }
        if (producerThread.isAlive()) {
            producerThread.interrupt();
        }
        if (consumerThread.isAlive()) {
            consumerThread.interrupt();
        }
        if (producerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Producer failed", producerError.get());
        }
        if (consumerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Consumer failed", consumerError.get());
        }
        if (!deadlinePassed) {
            LOG.info("+++ TEST passed! +++");
        } else {
            LOG.info("+++ TEST failed! +++");
        }
    } finally {
        client.deleteStream(streamName);
        client.shutdown();
        // stopping flink
        flink.after();
    }
}

Example 4 with AmazonKinesis

use of com.amazonaws.services.kinesis.AmazonKinesis in project beam by apache.

the class KinesisIOIT method setup.

@BeforeClass
public static void setup() throws Exception {
    PipelineOptionsFactory.register(KinesisTestOptions.class);
    options = TestPipeline.testingPipelineOptions().as(KinesisTestOptions.class);
    if (options.getUseLocalstack()) {
        setupLocalstack();
    }
    if (options.getCreateStream()) {
        AmazonKinesis kinesisClient = createKinesisClient();
        teardownTasks.add(kinesisClient::shutdown);
        createStream(kinesisClient);
        teardownTasks.add(() -> deleteStream(kinesisClient));
    }
}

Example 5 with AmazonKinesis

use of com.amazonaws.services.kinesis.AmazonKinesis in project voltdb by VoltDB.

the class KinesisStreamImporterConfig method discoverShards.

/**
     * connect to kinesis stream to discover the shards on the stream
     *
     * @param regionName The region name where the stream resides
     * @param streamName The kinesis stream name
     * @param accessKey The user access key
     * @param secretKey The user secret key
     * @param appName  The name of stream application
     * @return a list of shards
     */
public static List<Shard> discoverShards(String regionName, String streamName, String accessKey, String secretKey, String appName) {
    try {
        Region region = RegionUtils.getRegion(regionName);
        if (region != null) {
            final AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
            AmazonKinesis kinesisClient = new AmazonKinesisClient(credentials, getClientConfigWithUserAgent(appName));
            kinesisClient.setRegion(region);
            DescribeStreamResult result = kinesisClient.describeStream(streamName);
            if (!"ACTIVE".equals(result.getStreamDescription().getStreamStatus())) {
                throw new IllegalArgumentException("Kinesis stream " + streamName + " is not active.");
            }
            return result.getStreamDescription().getShards();
        }
    } catch (ResourceNotFoundException e) {
        LOGGER.warn("Kinesis stream " + streamName + " does not exist.", e);
    } catch (Exception e) {
        LOGGER.warn("Error found while describing the kinesis stream " + streamName, e);
    }
    return null;
}