Examples with CloudStorageSinkConfig org.apache.pulsar.io.jcloud.sink.CloudStorageSinkConfig used on opensource projects

Example 1 with CloudStorageSinkConfig

use of org.apache.pulsar.io.jcloud.sink.CloudStorageSinkConfig in project pulsar-io-cloud-storage by streamnative.

the class S3BlobWriterTest method initTest.

@Test
public void initTest() throws IOException {
    Map<String, Object> config = new HashMap<>();
    config.put("provider", PROVIDER_AWSS3V2);
    config.put("accessKeyId", "aws-s3");
    config.put("secretAccessKey", "aws-s3");
    config.put("bucket", "testbucket");
    config.put("region", "localhost");
    config.put("endpoint", "https://us-standard");
    config.put("pathPrefix", "pulsar/");
    config.put("formatType", "avro");
    config.put("partitionerType", "default");
    config.put("timePartitionPattern", "yyyy-MM-dd");
    config.put("timePartitionDuration", "2d");
    config.put("batchSize", 10);
    CloudStorageSinkConfig cloudStorageSinkConfig = CloudStorageSinkConfig.load(config);
    cloudStorageSinkConfig.validate();
    S3BlobWriter o = new S3BlobWriter(cloudStorageSinkConfig);
    o.close();
}

Example 2 with CloudStorageSinkConfig

use of org.apache.pulsar.io.jcloud.sink.CloudStorageSinkConfig in project pulsar-io-cloud-storage by streamnative.

the class ConnectorConfigTest method byteConfigTest.

@Test
public void byteConfigTest() throws IOException {
    Map<String, Object> config = new HashMap<>();
    config.put("provider", PROVIDER_AWSS3);
    config.put("accessKeyId", "aws-s3");
    config.put("secretAccessKey", "aws-s3");
    config.put("bucket", "testbucket");
    config.put("region", "localhost");
    config.put("endpoint", "us-standard");
    config.put("pathPrefix", "pulsar/");
    config.put("formatType", "bytes");
    config.put("partitionerType", "default");
    config.put("timePartitionPattern", "yyyy-MM-dd");
    config.put("timePartitionDuration", "2d");
    config.put("batchSize", 10);
    config.put("bytesFormatTypeSeparator", "0x10");
    CloudStorageSinkConfig cloudStorageSinkConfig = CloudStorageSinkConfig.load(config);
    cloudStorageSinkConfig.validate();
    Assert.assertEquals(PROVIDER_AWSS3, cloudStorageSinkConfig.getProvider());
    Assert.assertEquals(config.get("accessKeyId"), cloudStorageSinkConfig.getAccessKeyId());
    Assert.assertEquals(config.get("secretAccessKey"), cloudStorageSinkConfig.getSecretAccessKey());
    Assert.assertEquals(config.get("bucket"), cloudStorageSinkConfig.getBucket());
    Assert.assertEquals(config.get("region"), cloudStorageSinkConfig.getRegion());
    Assert.assertEquals(config.get("formatType"), cloudStorageSinkConfig.getFormatType());
    Assert.assertEquals(config.get("partitionerType"), cloudStorageSinkConfig.getPartitionerType());
    Assert.assertEquals(config.get("timePartitionPattern"), cloudStorageSinkConfig.getTimePartitionPattern());
    Assert.assertEquals(config.get("timePartitionDuration"), cloudStorageSinkConfig.getTimePartitionDuration());
    Assert.assertEquals(config.get("batchSize"), cloudStorageSinkConfig.getBatchSize());
    Assert.assertEquals(config.get("bytesFormatTypeSeparator"), cloudStorageSinkConfig.getBytesFormatTypeSeparator());
    Assert.assertEquals((int) config.get("batchSize") * 10, cloudStorageSinkConfig.getPendingQueueSize());
}

Example 3 with CloudStorageSinkConfig

use of org.apache.pulsar.io.jcloud.sink.CloudStorageSinkConfig in project pulsar-io-cloud-storage by streamnative.

the class ConnectorConfigTest method loadBasicConfigTest.

@Test
public void loadBasicConfigTest() throws IOException {
    Map<String, Object> config = new HashMap<>();
    config.put("provider", PROVIDER_AWSS3);
    config.put("accessKeyId", "aws-s3");
    config.put("secretAccessKey", "aws-s3");
    config.put("bucket", "testbucket");
    config.put("region", "localhost");
    config.put("endpoint", "us-standard");
    config.put("pathPrefix", "pulsar/");
    config.put("formatType", "avro");
    config.put("partitionerType", "default");
    config.put("timePartitionPattern", "yyyy-MM-dd");
    config.put("timePartitionDuration", "2d");
    config.put("batchSize", 10);
    CloudStorageSinkConfig cloudStorageSinkConfig = CloudStorageSinkConfig.load(config);
    cloudStorageSinkConfig.validate();
    Assert.assertEquals(PROVIDER_AWSS3, cloudStorageSinkConfig.getProvider());
    Assert.assertEquals(config.get("accessKeyId"), cloudStorageSinkConfig.getAccessKeyId());
    Assert.assertEquals(config.get("secretAccessKey"), cloudStorageSinkConfig.getSecretAccessKey());
    Assert.assertEquals(config.get("bucket"), cloudStorageSinkConfig.getBucket());
    Assert.assertEquals(config.get("region"), cloudStorageSinkConfig.getRegion());
    Assert.assertEquals(config.get("formatType"), cloudStorageSinkConfig.getFormatType());
    Assert.assertEquals(config.get("partitionerType"), cloudStorageSinkConfig.getPartitionerType());
    Assert.assertEquals(config.get("timePartitionPattern"), cloudStorageSinkConfig.getTimePartitionPattern());
    Assert.assertEquals(config.get("timePartitionDuration"), cloudStorageSinkConfig.getTimePartitionDuration());
    Assert.assertEquals(config.get("batchSize"), cloudStorageSinkConfig.getBatchSize());
    Assert.assertEquals((int) config.get("batchSize") * 10, cloudStorageSinkConfig.getPendingQueueSize());
}

Example 4 with CloudStorageSinkConfig

use of org.apache.pulsar.io.jcloud.sink.CloudStorageSinkConfig in project pulsar-io-cloud-storage by streamnative.

the class ConnectorConfigTest method loadFromMapCredentialFromSecretTest.

@Test
public final void loadFromMapCredentialFromSecretTest() throws IOException {
    Map<String, Object> config = new HashMap<>();
    config.put("provider", "aws-s3");
    config.put("bucket", "testbucket");
    config.put("region", "localhost");
    config.put("endpoint", "us-standard");
    config.put("formatType", "avro");
    config.put("partitionerType", "default");
    config.put("timePartitionPattern", "yyyy-MM-dd");
    config.put("timePartitionDuration", "2d");
    config.put("batchSize", 10);
    SinkContext sinkContext = Mockito.mock(SinkContext.class);
    Mockito.when(sinkContext.getSecret("accessKeyId")).thenReturn("myKeyId");
    Mockito.when(sinkContext.getSecret("secretAccessKey")).thenReturn("myAccessKey");
    CloudStorageSinkConfig sinkConfig = IOConfigUtils.loadWithSecrets(config, CloudStorageSinkConfig.class, sinkContext);
    Assert.assertNotNull(sinkConfig);
    Assert.assertEquals(sinkConfig.getProvider(), "aws-s3");
    Assert.assertEquals(sinkConfig.getBucket(), "testbucket");
    Assert.assertEquals(sinkConfig.getSecretAccessKey(), "myAccessKey");
    Assert.assertEquals(sinkConfig.getAccessKeyId(), "myKeyId");
}

Example 5 with CloudStorageSinkConfig

use of org.apache.pulsar.io.jcloud.sink.CloudStorageSinkConfig in project pulsar-io-cloud-storage by streamnative.

the class GoogleCloudStorageProviderTest method loadBasicConfigTest.

@Test
public void loadBasicConfigTest() throws IOException {
    Map<String, Object> config = new HashMap<>();
    config.put("provider", PROVIDER_GCS);
    config.put("gcsServiceAccountKeyFilePath", "/tmp/gcs.json");
    config.put("bucket", "testbucket");
    config.put("region", "localhost");
    config.put("endpoint", "us-standard");
    config.put("pathPrefix", "pulsar/");
    config.put("formatType", "avro");
    config.put("partitionerType", "default");
    config.put("timePartitionPattern", "yyyy-MM-dd");
    config.put("timePartitionDuration", "2d");
    config.put("batchSize", 10);
    CloudStorageSinkConfig cloudStorageSinkConfig = CloudStorageSinkConfig.load(config);
    cloudStorageSinkConfig.validate();
    Assert.assertEquals(PROVIDER_GCS, cloudStorageSinkConfig.getProvider());
    Assert.assertEquals(config.get("gcsServiceAccountKeyFilePath"), cloudStorageSinkConfig.getGcsServiceAccountKeyFilePath());
    Assert.assertEquals(config.get("bucket"), cloudStorageSinkConfig.getBucket());
    Assert.assertEquals(config.get("region"), cloudStorageSinkConfig.getRegion());
    Assert.assertEquals(config.get("formatType"), cloudStorageSinkConfig.getFormatType());
    Assert.assertEquals(config.get("partitionerType"), cloudStorageSinkConfig.getPartitionerType());
    Assert.assertEquals(config.get("timePartitionPattern"), cloudStorageSinkConfig.getTimePartitionPattern());
    Assert.assertEquals(config.get("timePartitionDuration"), cloudStorageSinkConfig.getTimePartitionDuration());
    Assert.assertEquals(config.get("batchSize"), cloudStorageSinkConfig.getBatchSize());
}