Examples with SourceContext org.apache.pulsar.io.core.SourceContext used on opensource projects

Example 6 with SourceContext

use of org.apache.pulsar.io.core.SourceContext in project pulsar by yahoo.

the class KinesisSourceConfigTests method loadFromMapCredentialFromSecretTest.

@Test
public final void loadFromMapCredentialFromSecretTest() throws IOException {
    Map<String, Object> map = new HashMap<String, Object>();
    map.put("awsEndpoint", "https://some.endpoint.aws");
    map.put("awsRegion", "us-east-1");
    map.put("awsKinesisStreamName", "my-stream");
    map.put("checkpointInterval", "30000");
    map.put("backoffTime", "4000");
    map.put("numRetries", "3");
    map.put("receiveQueueSize", 2000);
    map.put("applicationName", "My test application");
    map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON);
    map.put("startAtTime", DAY);
    SourceContext sourceContext = Mockito.mock(SourceContext.class);
    Mockito.when(sourceContext.getSecret("awsCredentialPluginParam")).thenReturn("{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
    KinesisSourceConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSourceConfig.class, sourceContext);
    assertNotNull(config);
    assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
    assertEquals(config.getAwsRegion(), "us-east-1");
    assertEquals(config.getAwsKinesisStreamName(), "my-stream");
    assertEquals(config.getAwsCredentialPluginParam(), "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
    assertEquals(config.getApplicationName(), "My test application");
    assertEquals(config.getCheckpointInterval(), 30000);
    assertEquals(config.getBackoffTime(), 4000);
    assertEquals(config.getNumRetries(), 3);
    assertEquals(config.getReceiveQueueSize(), 2000);
    assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
    Calendar cal = Calendar.getInstance();
    cal.setTime(config.getStartAtTime());
    ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC);
    ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC);
    assertEquals(actual, expected);
}

Example 7 with SourceContext

use of org.apache.pulsar.io.core.SourceContext in project pulsar by apache.

the class KafkaAbstractSourceTest method testInvalidConfigWillThrownException.

@Test
public void testInvalidConfigWillThrownException() throws Exception {
    KafkaAbstractSource source = new DummySource();
    SourceContext ctx = mock(SourceContext.class);
    Map<String, Object> config = new HashMap<>();
    Assert.ThrowingRunnable openAndClose = () -> {
        try {
            source.open(config, ctx);
            fail();
        } finally {
            source.close();
        }
    };
    expectThrows(NullPointerException.class, openAndClose);
    config.put("topic", "topic_1");
    expectThrows(NullPointerException.class, openAndClose);
    config.put("bootstrapServers", "localhost:8080");
    expectThrows(NullPointerException.class, openAndClose);
    config.put("groupId", "test-group");
    config.put("fetchMinBytes", -1);
    expectThrows(IllegalArgumentException.class, openAndClose);
    config.put("fetchMinBytes", 1000);
    config.put("autoCommitEnabled", true);
    config.put("autoCommitIntervalMs", -1);
    expectThrows(IllegalArgumentException.class, openAndClose);
    config.put("autoCommitIntervalMs", 100);
    config.put("sessionTimeoutMs", -1);
    expectThrows(IllegalArgumentException.class, openAndClose);
    config.put("sessionTimeoutMs", 10000);
    config.put("heartbeatIntervalMs", -100);
    expectThrows(IllegalArgumentException.class, openAndClose);
    config.put("heartbeatIntervalMs", 20000);
    expectThrows(IllegalArgumentException.class, openAndClose);
    config.put("heartbeatIntervalMs", 5000);
    config.put("autoOffsetReset", "some-value");
    expectThrows(IllegalArgumentException.class, openAndClose);
    config.put("autoOffsetReset", "earliest");
    source.open(config, ctx);
    source.close();
}

Example 8 with SourceContext

use of org.apache.pulsar.io.core.SourceContext in project pulsar by apache.

the class KinesisSourceConfigTests method loadFromMapTest.

@Test
public final void loadFromMapTest() throws IOException {
    Map<String, Object> map = new HashMap<String, Object>();
    map.put("awsEndpoint", "https://some.endpoint.aws");
    map.put("awsRegion", "us-east-1");
    map.put("awsKinesisStreamName", "my-stream");
    map.put("awsCredentialPluginParam", "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
    map.put("checkpointInterval", "30000");
    map.put("backoffTime", "4000");
    map.put("numRetries", "3");
    map.put("receiveQueueSize", 2000);
    map.put("applicationName", "My test application");
    map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON);
    map.put("startAtTime", DAY);
    SourceContext sourceContext = Mockito.mock(SourceContext.class);
    KinesisSourceConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSourceConfig.class, sourceContext);
    assertNotNull(config);
    assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
    assertEquals(config.getAwsRegion(), "us-east-1");
    assertEquals(config.getAwsKinesisStreamName(), "my-stream");
    assertEquals(config.getAwsCredentialPluginParam(), "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
    assertEquals(config.getApplicationName(), "My test application");
    assertEquals(config.getCheckpointInterval(), 30000);
    assertEquals(config.getBackoffTime(), 4000);
    assertEquals(config.getNumRetries(), 3);
    assertEquals(config.getReceiveQueueSize(), 2000);
    assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
    Calendar cal = Calendar.getInstance();
    cal.setTime(config.getStartAtTime());
    ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC);
    ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC);
    assertEquals(actual, expected);
}

Example 9 with SourceContext

use of org.apache.pulsar.io.core.SourceContext in project pulsar by apache.

the class KinesisSourceConfigTests method loadFromMapCredentialFromSecretTest.

@Test
public final void loadFromMapCredentialFromSecretTest() throws IOException {
    Map<String, Object> map = new HashMap<String, Object>();
    map.put("awsEndpoint", "https://some.endpoint.aws");
    map.put("awsRegion", "us-east-1");
    map.put("awsKinesisStreamName", "my-stream");
    map.put("checkpointInterval", "30000");
    map.put("backoffTime", "4000");
    map.put("numRetries", "3");
    map.put("receiveQueueSize", 2000);
    map.put("applicationName", "My test application");
    map.put("initialPositionInStream", InitialPositionInStream.TRIM_HORIZON);
    map.put("startAtTime", DAY);
    SourceContext sourceContext = Mockito.mock(SourceContext.class);
    Mockito.when(sourceContext.getSecret("awsCredentialPluginParam")).thenReturn("{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
    KinesisSourceConfig config = IOConfigUtils.loadWithSecrets(map, KinesisSourceConfig.class, sourceContext);
    assertNotNull(config);
    assertEquals(config.getAwsEndpoint(), "https://some.endpoint.aws");
    assertEquals(config.getAwsRegion(), "us-east-1");
    assertEquals(config.getAwsKinesisStreamName(), "my-stream");
    assertEquals(config.getAwsCredentialPluginParam(), "{\"accessKey\":\"myKey\",\"secretKey\":\"my-Secret\"}");
    assertEquals(config.getApplicationName(), "My test application");
    assertEquals(config.getCheckpointInterval(), 30000);
    assertEquals(config.getBackoffTime(), 4000);
    assertEquals(config.getNumRetries(), 3);
    assertEquals(config.getReceiveQueueSize(), 2000);
    assertEquals(config.getInitialPositionInStream(), InitialPositionInStream.TRIM_HORIZON);
    Calendar cal = Calendar.getInstance();
    cal.setTime(config.getStartAtTime());
    ZonedDateTime actual = ZonedDateTime.ofInstant(cal.toInstant(), ZoneOffset.UTC);
    ZonedDateTime expected = ZonedDateTime.ofInstant(DAY.toInstant(), ZoneOffset.UTC);
    assertEquals(actual, expected);
}