Examples with DLSN com.twitter.distributedlog.DLSN used on opensource projects

Example 26 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class ConsoleProxyMultiWriter method main.

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    final String streamList = args[1];
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("console-proxy-writer")).name("console-proxy-writer").thriftmux(true).finagleNameStr(finagleNameStr).build();
    String[] streamNameList = StringUtils.split(streamList, ',');
    DistributedLogMultiStreamWriter multiStreamWriter = DistributedLogMultiStreamWriter.newBuilder().streams(Lists.newArrayList(streamNameList)).bufferSize(0).client(client).flushIntervalMs(0).firstSpeculativeTimeoutMs(10000).maxSpeculativeTimeoutMs(20000).requestTimeoutMs(50000).build();
    ConsoleReader reader = new ConsoleReader();
    String line;
    while ((line = reader.readLine(PROMPT_MESSAGE)) != null) {
        multiStreamWriter.write(ByteBuffer.wrap(line.getBytes(UTF_8))).addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                Runtime.getRuntime().exit(0);
            }

            @Override
            public void onSuccess(DLSN value) {
            // done
            }
        });
    }
    multiStreamWriter.close();
    client.close();
}

Example 27 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class RecordGenerator method main.

public static void main(String[] args) throws Exception {
    if (3 != args.length) {
        System.out.println(HELP);
        return;
    }
    String finagleNameStr = args[0];
    final String streamName = args[1];
    double rate = Double.parseDouble(args[2]);
    RateLimiter limiter = RateLimiter.create(rate);
    DistributedLogClient client = DistributedLogClientBuilder.newBuilder().clientId(ClientId$.MODULE$.apply("record-generator")).name("record-generator").thriftmux(true).finagleNameStr(finagleNameStr).build();
    final CountDownLatch keepAliveLatch = new CountDownLatch(1);
    final AtomicLong numWrites = new AtomicLong(0);
    final AtomicBoolean running = new AtomicBoolean(true);
    while (running.get()) {
        limiter.acquire();
        String record = "record-" + System.currentTimeMillis();
        client.write(streamName, ByteBuffer.wrap(record.getBytes(UTF_8))).addEventListener(new FutureEventListener<DLSN>() {

            @Override
            public void onFailure(Throwable cause) {
                System.out.println("Encountered error on writing data");
                cause.printStackTrace(System.err);
                running.set(false);
                keepAliveLatch.countDown();
            }

            @Override
            public void onSuccess(DLSN value) {
                long numSuccesses = numWrites.incrementAndGet();
                if (numSuccesses % 100 == 0) {
                    System.out.println("Write " + numSuccesses + " records.");
                }
            }
        });
    }
    keepAliveLatch.await();
    client.close();
}

Example 28 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogServer method validateFailedAsLogRecordTooLong.

void validateFailedAsLogRecordTooLong(Future<DLSN> future) {
    try {
        DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
        fail("should have failed");
    } catch (DLException dle) {
        assertEquals(StatusCode.TOO_LARGE_RECORD, dle.getCode());
    } catch (Exception ex) {
        failDueToWrongException(ex);
    }
}

Example 29 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogServer method testHeartbeat.

@Test(timeout = 60000)
public void testHeartbeat() throws Exception {
    String name = "dlserver-heartbeat";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    for (long i = 1; i <= 10; i++) {
        logger.debug("Send heartbeat {} to stream {}.", i, name);
        dlClient.dlClient.check(name).get();
    }
    logger.debug("Write entry one to stream {}.", name);
    dlClient.dlClient.write(name, ByteBuffer.wrap("1".getBytes())).get();
    Thread.sleep(1000);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
    int numRead = 0;
    // eid=0 => control records
    // other 9 heartbeats will not trigger writing any control records.
    // eid=1 => user entry
    long startEntryId = 1;
    LogRecordWithDLSN r = reader.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(numRead + 1, i);
        assertEquals(r.getDlsn().compareTo(new DLSN(1, startEntryId, 0)), 0);
        ++numRead;
        ++startEntryId;
        r = reader.readNext(false);
    }
    assertEquals(1, numRead);
}

Example 30 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogServer method validateAllFailedAsCancelled.

int validateAllFailedAsCancelled(List<Future<DLSN>> futures, int start, int finish) {
    int failed = 0;
    for (int i = start; i < finish; i++) {
        Future<DLSN> future = futures.get(i);
        try {
            DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
            fail("future should have failed!");
        } catch (DLException cre) {
            ++failed;
        } catch (Exception ex) {
            failDueToWrongException(ex);
        }
    }
    return failed;
}