Examples with JsonlRecordFormatter org.komamitsu.fluency.aws.s3.recordformat.JsonlRecordFormatter used on opensource projects

Example 1 with JsonlRecordFormatter

use of org.komamitsu.fluency.aws.s3.recordformat.JsonlRecordFormatter in project fluency by komamitsu.

the class FluencyBuilderForAwsS3Test method buildWithCustomConfig.

@Test
void buildWithCustomConfig() {
    FluencyBuilderForAwsS3 builder = builderWithCustomConfig;
    AwsS3Sender sender = mock(AwsS3Sender.class);
    doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class));
    builder.setFormatType(FluencyBuilderForAwsS3.FormatType.JSONL);
    Fluency fluency = builder.build();
    assertEquals(fluency, this.fluency);
    ArgumentCaptor<AwsS3Sender.Config> configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class);
    verify(builder, times(1)).createSender(configArgumentCaptor.capture());
    AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue();
    assertEquals("https://foo.bar.org", senderConfig.getEndpoint());
    assertEquals(Region.US_EAST_1.toString(), senderConfig.getRegion());
    assertEquals("ACCESSKEYID", senderConfig.getAwsAccessKeyId());
    assertEquals("SECRETACCESSKEY", senderConfig.getAwsSecretAccessKey());
    assertEquals(4, senderConfig.getRetryMax());
    assertEquals(543, senderConfig.getRetryIntervalMs());
    assertEquals(65432, senderConfig.getMaxRetryIntervalMs());
    assertEquals(1.234f, senderConfig.getRetryFactor());
    assertEquals(99 * 1024, senderConfig.getWorkBufSize());
    assertFalse(senderConfig.isCompressionEnabled());
    ArgumentCaptor<RecordFormatter> recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class);
    ArgumentCaptor<Ingester> ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class);
    verify(builder, times(1)).buildFromIngester(recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture());
    RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue();
    assertTrue(recordFormatter instanceof JsonlRecordFormatter);
    AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue();
    assertEquals(sender, ingester.getSender());
    DefaultS3DestinationDecider destinationDecider = (DefaultS3DestinationDecider) ingester.getS3DestinationDecider();
    assertEquals("mydata", destinationDecider.getKeyPrefix());
    assertEquals(".zzz", destinationDecider.getKeySuffix());
    assertEquals(ZoneId.of("JST", SHORT_IDS), destinationDecider.getZoneId());
    ArgumentCaptor<Buffer.Config> bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class);
    ArgumentCaptor<Flusher.Config> flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class);
    verify(builder, times(1)).createFluency(eq(recordFormatter), eq(ingester), bufferConfigArgumentCaptor.capture(), flusherConfigArgumentCaptor.capture());
    Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue();
    assertEquals(42 * 1024 * 1024, bufferConfig.getMaxBufferSize());
    assertEquals(2 * 1024 * 1024, bufferConfig.getChunkInitialSize());
    assertEquals(9 * 1024 * 1024, bufferConfig.getChunkRetentionSize());
    assertEquals(1234, bufferConfig.getChunkRetentionTimeMillis());
}

Example 2 with JsonlRecordFormatter

use of org.komamitsu.fluency.aws.s3.recordformat.JsonlRecordFormatter in project fluency by komamitsu.

the class FluencyBuilderForAwsS3 method createRecordFormatter.

private AwsS3RecordFormatter createRecordFormatter() {
    if (formatType == null) {
        throw new IllegalArgumentException("format type must be set");
    }
    AwsS3RecordFormatter recordFormatter;
    switch(getFormatType()) {
        case MESSAGE_PACK:
            recordFormatter = new MessagePackRecordFormatter();
            break;
        case JSONL:
            recordFormatter = new JsonlRecordFormatter();
            break;
        case CSV:
            CsvRecordFormatter.Config config = new CsvRecordFormatter.Config();
            config.setColumnNames(getFormatCsvColumnNames());
            recordFormatter = new CsvRecordFormatter(config);
            break;
        default:
            throw new IllegalArgumentException("Unexpected format type: " + getFormatType());
    }
    return recordFormatter;
}