Examples with Ingester org.komamitsu.fluency.ingester.Ingester used on opensource projects

Example 1 with Ingester

use of org.komamitsu.fluency.ingester.Ingester in project fluency by komamitsu.

the class FluentdIngesterTest method close.

@Test
void close() throws IOException {
    Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender);
    ingester.close();
    verify(fluentdSender, times(1)).close();
}

Example 2 with Ingester

use of org.komamitsu.fluency.ingester.Ingester in project fluency by komamitsu.

the class FluentdIngesterTest method ingestWithAck.

@Test
void ingestWithAck() throws IOException {
    FluentdIngester.Config config = new FluentdIngester.Config();
    config.setAckResponseMode(true);
    Ingester ingester = new FluentdIngester(config, fluentdSender);
    ingester.ingest(TAG, ByteBuffer.wrap(DATA));
    ArgumentCaptor<String> ackTokenArgumentCaptor = ArgumentCaptor.forClass(String.class);
    verify(fluentdSender, times(1)).sendWithAck(byteBuffersArgumentCaptor.capture(), ackTokenArgumentCaptor.capture());
    List<ByteBuffer> byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0);
    byte[] ingested = getIngestedData(byteBuffers);
    MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested);
    ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue();
    assertEquals(3, arrayValue.size());
    assertEquals(TAG, arrayValue.get(0).asStringValue().asString());
    assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray());
    Map<Value, Value> options = arrayValue.get(2).asMapValue().map();
    assertEquals(2, options.size());
    assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt());
    String ackToken = options.get(ValueFactory.newString("chunk")).asRawValue().asString();
    List<String> ackTokenArgumentCaptorAllValues = ackTokenArgumentCaptor.getAllValues();
    assertEquals(1, ackTokenArgumentCaptorAllValues.size());
    assertEquals(ackToken, ackTokenArgumentCaptorAllValues.get(0));
}

Example 3 with Ingester

use of org.komamitsu.fluency.ingester.Ingester in project fluency by komamitsu.

the class BufferTest method setUp.

@BeforeEach
void setUp() {
    recordFormatter = new JsonRecordFormatter();
    ingester = mock(Ingester.class);
    bufferConfig = new Buffer.Config();
}

Example 4 with Ingester

use of org.komamitsu.fluency.ingester.Ingester in project fluency by komamitsu.

the class BufferTest method flush.

@Test
void flush() throws IOException {
    Buffer buffer = new Buffer(bufferConfig, recordFormatter);
    Ingester ingester = mock(Ingester.class);
    buffer.flush(ingester, false);
    verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class));
    Map<String, Object> data = new HashMap<>();
    data.put("name", "komamitsu");
    buffer.append("foodb.bartbl", 1420070400, data);
    buffer.flush(ingester, false);
    verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class));
    buffer.flush(ingester, true);
    verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class));
}

Example 5 with Ingester

use of org.komamitsu.fluency.ingester.Ingester in project fluency by komamitsu.

the class FluencyBuilderForAwsS3Test method buildWithCustomConfig.

@Test
void buildWithCustomConfig() {
    FluencyBuilderForAwsS3 builder = builderWithCustomConfig;
    AwsS3Sender sender = mock(AwsS3Sender.class);
    doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class));
    builder.setFormatType(FluencyBuilderForAwsS3.FormatType.JSONL);
    Fluency fluency = builder.build();
    assertEquals(fluency, this.fluency);
    ArgumentCaptor<AwsS3Sender.Config> configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class);
    verify(builder, times(1)).createSender(configArgumentCaptor.capture());
    AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue();
    assertEquals("https://foo.bar.org", senderConfig.getEndpoint());
    assertEquals(Region.US_EAST_1.toString(), senderConfig.getRegion());
    assertEquals("ACCESSKEYID", senderConfig.getAwsAccessKeyId());
    assertEquals("SECRETACCESSKEY", senderConfig.getAwsSecretAccessKey());
    assertEquals(4, senderConfig.getRetryMax());
    assertEquals(543, senderConfig.getRetryIntervalMs());
    assertEquals(65432, senderConfig.getMaxRetryIntervalMs());
    assertEquals(1.234f, senderConfig.getRetryFactor());
    assertEquals(99 * 1024, senderConfig.getWorkBufSize());
    assertFalse(senderConfig.isCompressionEnabled());
    ArgumentCaptor<RecordFormatter> recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class);
    ArgumentCaptor<Ingester> ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class);
    verify(builder, times(1)).buildFromIngester(recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture());
    RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue();
    assertTrue(recordFormatter instanceof JsonlRecordFormatter);
    AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue();
    assertEquals(sender, ingester.getSender());
    DefaultS3DestinationDecider destinationDecider = (DefaultS3DestinationDecider) ingester.getS3DestinationDecider();
    assertEquals("mydata", destinationDecider.getKeyPrefix());
    assertEquals(".zzz", destinationDecider.getKeySuffix());
    assertEquals(ZoneId.of("JST", SHORT_IDS), destinationDecider.getZoneId());
    ArgumentCaptor<Buffer.Config> bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class);
    ArgumentCaptor<Flusher.Config> flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class);
    verify(builder, times(1)).createFluency(eq(recordFormatter), eq(ingester), bufferConfigArgumentCaptor.capture(), flusherConfigArgumentCaptor.capture());
    Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue();
    assertEquals(42 * 1024 * 1024, bufferConfig.getMaxBufferSize());
    assertEquals(2 * 1024 * 1024, bufferConfig.getChunkInitialSize());
    assertEquals(9 * 1024 * 1024, bufferConfig.getChunkRetentionSize());
    assertEquals(1234, bufferConfig.getChunkRetentionTimeMillis());
}