Examples with AwsS3RecordFormatter org.komamitsu.fluency.aws.s3.recordformat.AwsS3RecordFormatter used on opensource projects

Example 1 with AwsS3RecordFormatter

use of org.komamitsu.fluency.aws.s3.recordformat.AwsS3RecordFormatter in project fluency by komamitsu.

the class FluencyBuilderForAwsS3 method createIngester.

private AwsS3Ingester createIngester(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) {
    S3DestinationDecider s3DestinationDecider;
    if (getCustomS3DestinationDecider() == null) {
        DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config();
        if (getS3KeyPrefix() != null) {
            config.setKeyPrefix(getS3KeyPrefix());
        }
        if (getS3KeySuffix() != null) {
            config.setKeySuffix(getS3KeySuffix());
        } else {
            config.setKeySuffix(defaultKeyPrefix(recordFormatter) + (isCompressionEnabled() ? ".gz" : ""));
        }
        if (getS3KeyTimeZoneId() != null) {
            config.setZoneId(getS3KeyTimeZoneId());
        }
        s3DestinationDecider = new DefaultS3DestinationDecider(config);
    } else {
        s3DestinationDecider = getCustomS3DestinationDecider();
    }
    AwsS3Sender sender = createSender(senderConfig);
    return new AwsS3Ingester(sender, s3DestinationDecider);
}

Example 2 with AwsS3RecordFormatter

use of org.komamitsu.fluency.aws.s3.recordformat.AwsS3RecordFormatter in project fluency by komamitsu.

the class FluencyBuilderForAwsS3 method createRecordFormatter.

private AwsS3RecordFormatter createRecordFormatter() {
    if (formatType == null) {
        throw new IllegalArgumentException("format type must be set");
    }
    AwsS3RecordFormatter recordFormatter;
    switch(getFormatType()) {
        case MESSAGE_PACK:
            recordFormatter = new MessagePackRecordFormatter();
            break;
        case JSONL:
            recordFormatter = new JsonlRecordFormatter();
            break;
        case CSV:
            CsvRecordFormatter.Config config = new CsvRecordFormatter.Config();
            config.setColumnNames(getFormatCsvColumnNames());
            recordFormatter = new CsvRecordFormatter(config);
            break;
        default:
            throw new IllegalArgumentException("Unexpected format type: " + getFormatType());
    }
    return recordFormatter;
}

Example 3 with AwsS3RecordFormatter

use of org.komamitsu.fluency.aws.s3.recordformat.AwsS3RecordFormatter in project fluency by komamitsu.

the class FluencyBuilderForAwsS3Test method buildWithDefaultConfig.

@ParameterizedTest
@EnumSource(FluencyBuilderForAwsS3.FormatType.class)
void buildWithDefaultConfig(FluencyBuilderForAwsS3.FormatType formatType) {
    FluencyBuilderForAwsS3 builder = builderWithDefaultConfig;
    AwsS3Sender sender = mock(AwsS3Sender.class);
    doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class));
    builder.setFormatType(formatType);
    Fluency fluency = builder.build();
    assertEquals(fluency, this.fluency);
    ArgumentCaptor<AwsS3Sender.Config> configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class);
    verify(builder, times(1)).createSender(configArgumentCaptor.capture());
    AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue();
    assertNull(senderConfig.getEndpoint());
    assertNull(senderConfig.getRegion());
    assertNull(senderConfig.getAwsAccessKeyId());
    assertNull(senderConfig.getAwsSecretAccessKey());
    assertEquals(10, senderConfig.getRetryMax());
    assertEquals(1000, senderConfig.getRetryIntervalMs());
    assertEquals(30000, senderConfig.getMaxRetryIntervalMs());
    assertEquals(2.0, senderConfig.getRetryFactor());
    assertEquals(8192, senderConfig.getWorkBufSize());
    assertTrue(senderConfig.isCompressionEnabled());
    ArgumentCaptor<RecordFormatter> recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class);
    ArgumentCaptor<Ingester> ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class);
    verify(builder, times(1)).buildFromIngester(recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture());
    RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue();
    Class<? extends AwsS3RecordFormatter> expectedAwsS3RecordFormatter = null;
    String expectedS3KeySuffix = null;
    switch(formatType) {
        case MESSAGE_PACK:
            expectedAwsS3RecordFormatter = MessagePackRecordFormatter.class;
            expectedS3KeySuffix = ".msgpack.gz";
            break;
        case JSONL:
            expectedAwsS3RecordFormatter = JsonlRecordFormatter.class;
            expectedS3KeySuffix = ".jsonl.gz";
            break;
        case CSV:
            expectedAwsS3RecordFormatter = CsvRecordFormatter.class;
            expectedS3KeySuffix = ".csv.gz";
            break;
    }
    assertEquals(expectedAwsS3RecordFormatter, recordFormatter.getClass());
    AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue();
    assertEquals(sender, ingester.getSender());
    DefaultS3DestinationDecider destinationDecider = (DefaultS3DestinationDecider) ingester.getS3DestinationDecider();
    assertNull(destinationDecider.getKeyPrefix());
    assertEquals(expectedS3KeySuffix, destinationDecider.getKeySuffix());
    assertEquals(UTC, destinationDecider.getZoneId());
    ArgumentCaptor<Buffer.Config> bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class);
    ArgumentCaptor<Flusher.Config> flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class);
    verify(builder, times(1)).createFluency(eq(recordFormatter), eq(ingester), bufferConfigArgumentCaptor.capture(), flusherConfigArgumentCaptor.capture());
    Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue();
    assertEquals(512 * 1024 * 1024, bufferConfig.getMaxBufferSize());
    assertEquals(4 * 1024 * 1024, bufferConfig.getChunkInitialSize());
    assertEquals(64 * 1024 * 1024, bufferConfig.getChunkRetentionSize());
    assertEquals(30 * 1000, bufferConfig.getChunkRetentionTimeMillis());
}