Examples with DistributedLogClient com.twitter.distributedlog.service.DistributedLogClient used on opensource projects

Example 6 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class ConsoleProxyWriter method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    final String streamName = args[1];
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("console-proxy-writer")).name("console-proxy-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    ConsoleReader reader = new ConsoleReader();
    String line;
    while ((line = reader.readLine(PROMPT_MESSAGE)) != null) {
        client.write(streamName, ByteBuffer.wrap(line.getBytes(UTF_8))).addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN value) {
            // done
            }
        });
    }
    client.close();
}

Example 7 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testFlushWhenBufferIsFull.

@Test(timeout = 20000)
public void testFlushWhenBufferIsFull() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())).thenReturn(Future.value(new DLSN(1L, 1L, 999L)));
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(100000).maxSpeculativeTimeoutMs(200000).speculativeBackoffMultiplier(2).requestTimeoutMs(500000).flushIntervalMs(0).bufferSize(0).scheduler(executorService).build();
    ByteBuffer buffer = ByteBuffer.wrap("test".getBytes(UTF_8));
    writer.write(buffer);
    verify(client, times(1)).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    writer.close();
}

Example 8 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testWriteTooLargeRecord.

@Test(timeout = 20000)
public void testWriteTooLargeRecord() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(100000).maxSpeculativeTimeoutMs(200000).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(0).bufferSize(0).build();
    byte[] data = new byte[LogRecord.MAX_LOGRECORD_SIZE + 10];
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    assertTrue(writeFuture.isDefined());
    try {
        Await.result(writeFuture);
        fail("Should fail on writing too long record");
    } catch (LogRecordTooLongException lrtle) {
    // expected
    }
    writer.close();
}

Example 9 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testSpeculativeWrite.

@Test(timeout = 20000)
public void testSpeculativeWrite() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(10).maxSpeculativeTimeoutMs(20).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(0).bufferSize(0).build();
    final String secondStream = writer.getStream(1);
    final DLSN dlsn = new DLSN(99L, 88L, 0L);
    Mockito.doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Object[] arguments = invocation.getArguments();
            String stream = (String) arguments[0];
            if (stream.equals(secondStream)) {
                return Future.value(dlsn);
            } else {
                return new Promise<DLSN>();
            }
        }
    }).when(client).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    byte[] data = "test-test".getBytes(UTF_8);
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    DLSN writeDLSN = Await.result(writeFuture);
    assertEquals(dlsn, writeDLSN);
    writer.close();
}

Example 10 with DistributedLogClient

use of com.twitter.distributedlog.service.DistributedLogClient in project distributedlog by twitter.

the class AtomicWriter method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    String streamName = args[1];
    String[] messages = new String[args.length - 2];
    System.arraycopy(args, 2, messages, 0, messages.length);
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("atomic-writer")).name("atomic-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    final LogRecordSet.Writer recordSetWriter = LogRecordSet.newWriter(16 * 1024, Type.NONE);
    List<Future<DLSN>> writeFutures = Lists.newArrayListWithExpectedSize(messages.length);
    for (String msg : messages) {
        final String message = msg;
        ByteBuffer msgBuf = ByteBuffer.wrap(msg.getBytes(UTF_8));
        Promise<DLSN> writeFuture = new Promise<DLSN>();
        writeFuture.addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN dlsn) {
                System.out.println("Write '" + message + "' as record " + dlsn);
            }
        });
        recordSetWriter.writeRecord(msgBuf, writeFuture);
        writeFutures.add(writeFuture);
    }
    FutureUtils.result(client.writeRecordSet(streamName, recordSetWriter).addEventListener(new FutureEventListener<DLSN>() {

        @Override
        public void onFailure(Throwable cause) {
            recordSetWriter.abortTransmit(cause);
            System.out.println("Encountered error on writing data");
            cause.printStackTrace(System.err);
            Runtime.getRuntime().exit(0);
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            recordSetWriter.completeTransmit(dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), dlsn.getSlotId());
        }
    }));
    FutureUtils.result(Future.collect(writeFutures));
    client.close();
}