Examples with DistributedLogClient com.twitter.distributedlog.service.DistributedLogClient used on opensource projects

Example 1 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testFailRequestAfterRetriedAllStreams.

@Test(timeout = 20000)
public void testFailRequestAfterRetriedAllStreams() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())).thenReturn(new Promise<DLSN>());
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(10).maxSpeculativeTimeoutMs(20).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(10).bufferSize(Integer.MAX_VALUE).build();
    byte[] data = "test-test".getBytes(UTF_8);
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    try {
        Await.result(writeFuture);
        fail("Should fail the request after retries all streams");
    } catch (IndividualRequestTimeoutException e) {
        long timeoutMs = e.timeout().inMilliseconds();
        assertTrue(timeoutMs >= (10 + 20) && timeoutMs < 5000000);
    }
    writer.close();
}

Example 2 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testPeriodicalFlush.

@Test(timeout = 20000)
public void testPeriodicalFlush() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(10).maxSpeculativeTimeoutMs(20).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(10).bufferSize(Integer.MAX_VALUE).build();
    final DLSN dlsn = new DLSN(99L, 88L, 0L);
    Mockito.doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            return Future.value(dlsn);
        }
    }).when(client).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    byte[] data = "test-test".getBytes(UTF_8);
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    DLSN writeDLSN = Await.result(writeFuture);
    assertEquals(dlsn, writeDLSN);
    writer.close();
}

Example 3 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testFlushWhenExceedMaxLogRecordSetSize.

@Test(timeout = 20000)
public void testFlushWhenExceedMaxLogRecordSetSize() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())).thenReturn(Future.value(new DLSN(1L, 1L, 999L)));
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(100000).maxSpeculativeTimeoutMs(200000).speculativeBackoffMultiplier(2).requestTimeoutMs(500000).flushIntervalMs(0).bufferSize(Integer.MAX_VALUE).scheduler(executorService).build();
    byte[] data = new byte[LogRecord.MAX_LOGRECORD_SIZE - 3 * 100];
    ByteBuffer buffer1 = ByteBuffer.wrap(data);
    writer.write(buffer1);
    verify(client, times(0)).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    LogRecordSet.Writer recordSetWriter1 = writer.getLogRecordSetWriter();
    assertEquals(1, recordSetWriter1.getNumRecords());
    assertEquals(LogRecordSet.HEADER_LEN + 4 + data.length, recordSetWriter1.getNumBytes());
    ByteBuffer buffer2 = ByteBuffer.wrap(data);
    writer.write(buffer2);
    verify(client, times(1)).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    LogRecordSet.Writer recordSetWriter2 = writer.getLogRecordSetWriter();
    assertEquals(1, recordSetWriter2.getNumRecords());
    assertEquals(LogRecordSet.HEADER_LEN + 4 + data.length, recordSetWriter2.getNumBytes());
    assertTrue(recordSetWriter1 != recordSetWriter2);
    writer.close();
}

Example 4 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class ConsoleProxyPartitionedMultiWriter method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    final String streamList = args[1];
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("console-proxy-writer")).name("console-proxy-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    String[] streamNameList = StringUtils.split(streamList, ',');
    PartitionedWriter<Integer, String> partitionedWriter = new PartitionedWriter<Integer, String>(streamNameList, new IntPartitioner(), client);
    ConsoleReader reader = new ConsoleReader();
    String line;
    while ((line = reader.readLine(PROMPT_MESSAGE)) != null) {
        String[] parts = StringUtils.split(line, ':');
        if (parts.length != 2) {
            System.out.println("Invalid input. Needs 'KEY:VALUE'");
            continue;
        }
        int key;
        try {
            key = Integer.parseInt(parts[0]);
        } catch (NumberFormatException nfe) {
            System.out.println("Invalid input. Needs 'KEY:VALUE'");
            continue;
        }
        String value = parts[1];
        partitionedWriter.write(key, value).addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN value) {
            // done
            }
        });
    }
    client.close();
}

Example 5 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class ConsoleProxyRRMultiWriter method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    final String streamList = args[1];
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("console-proxy-writer")).name("console-proxy-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    String[] streamNameList = StringUtils.split(streamList, ',');
    RRMultiWriter<Integer, String> writer = new RRMultiWriter(streamNameList, client);
    ConsoleReader reader = new ConsoleReader();
    String line;
    while ((line = reader.readLine(PROMPT_MESSAGE)) != null) {
        writer.write(line).addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN value) {
            // done
            }
        });
    }
    client.close();
}