Examples with DLSN com.twitter.distributedlog.DLSN used on opensource projects

Example 21 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testFlushWhenBufferIsFull.

@Test(timeout = 20000)
public void testFlushWhenBufferIsFull() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    when(client.writeRecordSet((String) any(), (LogRecordSetBuffer) any())).thenReturn(Future.value(new DLSN(1L, 1L, 999L)));
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(100000).maxSpeculativeTimeoutMs(200000).speculativeBackoffMultiplier(2).requestTimeoutMs(500000).flushIntervalMs(0).bufferSize(0).scheduler(executorService).build();
    ByteBuffer buffer = ByteBuffer.wrap("test".getBytes(UTF_8));
    writer.write(buffer);
    verify(client, times(1)).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    writer.close();
}

Example 22 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testWriteTooLargeRecord.

@Test(timeout = 20000)
public void testWriteTooLargeRecord() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(100000).maxSpeculativeTimeoutMs(200000).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(0).bufferSize(0).build();
    byte[] data = new byte[LogRecord.MAX_LOGRECORD_SIZE + 10];
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    assertTrue(writeFuture.isDefined());
    try {
        Await.result(writeFuture);
        fail("Should fail on writing too long record");
    } catch (LogRecordTooLongException lrtle) {
    // expected
    }
    writer.close();
}

Example 23 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogMultiStreamWriter method testSpeculativeWrite.

@Test(timeout = 20000)
public void testSpeculativeWrite() throws Exception {
    DistributedLogClient client = mock(DistributedLogClient.class);
    DistributedLogMultiStreamWriter writer = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList("stream1", "stream2")).client(client).compressionCodec(CompressionCodec.Type.LZ4).firstSpeculativeTimeoutMs(10).maxSpeculativeTimeoutMs(20).speculativeBackoffMultiplier(2).requestTimeoutMs(5000000).flushIntervalMs(0).bufferSize(0).build();
    final String secondStream = writer.getStream(1);
    final DLSN dlsn = new DLSN(99L, 88L, 0L);
    Mockito.doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Object[] arguments = invocation.getArguments();
            String stream = (String) arguments[0];
            if (stream.equals(secondStream)) {
                return Future.value(dlsn);
            } else {
                return new Promise<DLSN>();
            }
        }
    }).when(client).writeRecordSet((String) any(), (LogRecordSetBuffer) any());
    byte[] data = "test-test".getBytes(UTF_8);
    ByteBuffer buffer = ByteBuffer.wrap(data);
    Future<DLSN> writeFuture = writer.write(buffer);
    DLSN writeDLSN = Await.result(writeFuture);
    assertEquals(dlsn, writeDLSN);
    writer.close();
}

Example 24 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestStreamOp method testResponseSucceededThenFailed.

@Test(timeout = 60000)
public void testResponseSucceededThenFailed() throws Exception {
    AsyncLogWriter writer = mock(AsyncLogWriter.class);
    when(writer.write((LogRecord) any())).thenReturn(Future.value(new DLSN(1, 2, 3)));
    when(writer.getStreamName()).thenReturn("test");
    WriteOp writeOp = getWriteOp();
    writeOp.execute(writer, new Sequencer() {

        public long nextId() {
            return 0;
        }
    }, new Object());
    writeOp.fail(new InternalServerException("test2"));
    WriteResponse response = Await.result(writeOp.result());
    assertEquals(StatusCode.SUCCESS, response.getHeader().getCode());
}

Example 25 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class AtomicWriter method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    String streamName = args[1];
    String[] messages = new String[args.length - 2];
    System.arraycopy(args, 2, messages, 0, messages.length);
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("atomic-writer")).name("atomic-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    final LogRecordSet.Writer recordSetWriter = LogRecordSet.newWriter(16 * 1024, Type.NONE);
    List<Future<DLSN>> writeFutures = Lists.newArrayListWithExpectedSize(messages.length);
    for (String msg : messages) {
        final String message = msg;
        ByteBuffer msgBuf = ByteBuffer.wrap(msg.getBytes(UTF_8));
        Promise<DLSN> writeFuture = new Promise<DLSN>();
        writeFuture.addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN dlsn) {
                System.out.println("Write '" + message + "' as record " + dlsn);
            }
        });
        recordSetWriter.writeRecord(msgBuf, writeFuture);
        writeFutures.add(writeFuture);
    }
    FutureUtils.result(client.writeRecordSet(streamName, recordSetWriter).addEventListener(new FutureEventListener<DLSN>() {

        @Override
        public void onFailure(Throwable cause) {
            recordSetWriter.abortTransmit(cause);
            System.out.println("Encountered error on writing data");
            cause.printStackTrace(System.err);
            Runtime.getRuntime().exit(0);
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            recordSetWriter.completeTransmit(dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), dlsn.getSlotId());
        }
    }));
    FutureUtils.result(Future.collect(writeFutures));
    client.close();
}