Examples with AwsS3Ingester org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester used on opensource projects

Example 1 with AwsS3Ingester

use of org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester in project fluency by komamitsu.

the class FluencyBuilderForAwsS3 method createIngester.

private AwsS3Ingester createIngester(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) {
    S3DestinationDecider s3DestinationDecider;
    if (getCustomS3DestinationDecider() == null) {
        DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config();
        if (getS3KeyPrefix() != null) {
            config.setKeyPrefix(getS3KeyPrefix());
        }
        if (getS3KeySuffix() != null) {
            config.setKeySuffix(getS3KeySuffix());
        } else {
            config.setKeySuffix(defaultKeyPrefix(recordFormatter) + (isCompressionEnabled() ? ".gz" : ""));
        }
        if (getS3KeyTimeZoneId() != null) {
            config.setZoneId(getS3KeyTimeZoneId());
        }
        s3DestinationDecider = new DefaultS3DestinationDecider(config);
    } else {
        s3DestinationDecider = getCustomS3DestinationDecider();
    }
    AwsS3Sender sender = createSender(senderConfig);
    return new AwsS3Ingester(sender, s3DestinationDecider);
}

Example 2 with AwsS3Ingester

use of org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester in project fluency by komamitsu.

the class FluencyBuilderForAwsS3Test method buildWithCustomConfig.

@Test
void buildWithCustomConfig() {
    FluencyBuilderForAwsS3 builder = builderWithCustomConfig;
    AwsS3Sender sender = mock(AwsS3Sender.class);
    doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class));
    builder.setFormatType(FluencyBuilderForAwsS3.FormatType.JSONL);
    Fluency fluency = builder.build();
    assertEquals(fluency, this.fluency);
    ArgumentCaptor<AwsS3Sender.Config> configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class);
    verify(builder, times(1)).createSender(configArgumentCaptor.capture());
    AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue();
    assertEquals("https://foo.bar.org", senderConfig.getEndpoint());
    assertEquals(Region.US_EAST_1.toString(), senderConfig.getRegion());
    assertEquals("ACCESSKEYID", senderConfig.getAwsAccessKeyId());
    assertEquals("SECRETACCESSKEY", senderConfig.getAwsSecretAccessKey());
    assertEquals(4, senderConfig.getRetryMax());
    assertEquals(543, senderConfig.getRetryIntervalMs());
    assertEquals(65432, senderConfig.getMaxRetryIntervalMs());
    assertEquals(1.234f, senderConfig.getRetryFactor());
    assertEquals(99 * 1024, senderConfig.getWorkBufSize());
    assertFalse(senderConfig.isCompressionEnabled());
    ArgumentCaptor<RecordFormatter> recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class);
    ArgumentCaptor<Ingester> ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class);
    verify(builder, times(1)).buildFromIngester(recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture());
    RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue();
    assertTrue(recordFormatter instanceof JsonlRecordFormatter);
    AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue();
    assertEquals(sender, ingester.getSender());
    DefaultS3DestinationDecider destinationDecider = (DefaultS3DestinationDecider) ingester.getS3DestinationDecider();
    assertEquals("mydata", destinationDecider.getKeyPrefix());
    assertEquals(".zzz", destinationDecider.getKeySuffix());
    assertEquals(ZoneId.of("JST", SHORT_IDS), destinationDecider.getZoneId());
    ArgumentCaptor<Buffer.Config> bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class);
    ArgumentCaptor<Flusher.Config> flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class);
    verify(builder, times(1)).createFluency(eq(recordFormatter), eq(ingester), bufferConfigArgumentCaptor.capture(), flusherConfigArgumentCaptor.capture());
    Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue();
    assertEquals(42 * 1024 * 1024, bufferConfig.getMaxBufferSize());
    assertEquals(2 * 1024 * 1024, bufferConfig.getChunkInitialSize());
    assertEquals(9 * 1024 * 1024, bufferConfig.getChunkRetentionSize());
    assertEquals(1234, bufferConfig.getChunkRetentionTimeMillis());
}

Example 3 with AwsS3Ingester

use of org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester in project fluency by komamitsu.

the class FluencyBuilderForAwsS3Test method buildWithDefaultConfig.

@ParameterizedTest
@EnumSource(FluencyBuilderForAwsS3.FormatType.class)
void buildWithDefaultConfig(FluencyBuilderForAwsS3.FormatType formatType) {
    FluencyBuilderForAwsS3 builder = builderWithDefaultConfig;
    AwsS3Sender sender = mock(AwsS3Sender.class);
    doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class));
    builder.setFormatType(formatType);
    Fluency fluency = builder.build();
    assertEquals(fluency, this.fluency);
    ArgumentCaptor<AwsS3Sender.Config> configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class);
    verify(builder, times(1)).createSender(configArgumentCaptor.capture());
    AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue();
    assertNull(senderConfig.getEndpoint());
    assertNull(senderConfig.getRegion());
    assertNull(senderConfig.getAwsAccessKeyId());
    assertNull(senderConfig.getAwsSecretAccessKey());
    assertEquals(10, senderConfig.getRetryMax());
    assertEquals(1000, senderConfig.getRetryIntervalMs());
    assertEquals(30000, senderConfig.getMaxRetryIntervalMs());
    assertEquals(2.0, senderConfig.getRetryFactor());
    assertEquals(8192, senderConfig.getWorkBufSize());
    assertTrue(senderConfig.isCompressionEnabled());
    ArgumentCaptor<RecordFormatter> recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class);
    ArgumentCaptor<Ingester> ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class);
    verify(builder, times(1)).buildFromIngester(recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture());
    RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue();
    Class<? extends AwsS3RecordFormatter> expectedAwsS3RecordFormatter = null;
    String expectedS3KeySuffix = null;
    switch(formatType) {
        case MESSAGE_PACK:
            expectedAwsS3RecordFormatter = MessagePackRecordFormatter.class;
            expectedS3KeySuffix = ".msgpack.gz";
            break;
        case JSONL:
            expectedAwsS3RecordFormatter = JsonlRecordFormatter.class;
            expectedS3KeySuffix = ".jsonl.gz";
            break;
        case CSV:
            expectedAwsS3RecordFormatter = CsvRecordFormatter.class;
            expectedS3KeySuffix = ".csv.gz";
            break;
    }
    assertEquals(expectedAwsS3RecordFormatter, recordFormatter.getClass());
    AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue();
    assertEquals(sender, ingester.getSender());
    DefaultS3DestinationDecider destinationDecider = (DefaultS3DestinationDecider) ingester.getS3DestinationDecider();
    assertNull(destinationDecider.getKeyPrefix());
    assertEquals(expectedS3KeySuffix, destinationDecider.getKeySuffix());
    assertEquals(UTC, destinationDecider.getZoneId());
    ArgumentCaptor<Buffer.Config> bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class);
    ArgumentCaptor<Flusher.Config> flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class);
    verify(builder, times(1)).createFluency(eq(recordFormatter), eq(ingester), bufferConfigArgumentCaptor.capture(), flusherConfigArgumentCaptor.capture());
    Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue();
    assertEquals(512 * 1024 * 1024, bufferConfig.getMaxBufferSize());
    assertEquals(4 * 1024 * 1024, bufferConfig.getChunkInitialSize());
    assertEquals(64 * 1024 * 1024, bufferConfig.getChunkRetentionSize());
    assertEquals(30 * 1000, bufferConfig.getChunkRetentionTimeMillis());
}