Examples with DistributedLogManager com.twitter.distributedlog.DistributedLogManager used on opensource projects

Example 16 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class SyncReaderBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String streamName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    OpStatsLogger openReaderStats = statsLogger.getOpStatsLogger("open_reader");
    OpStatsLogger nonBlockingReadStats = statsLogger.getOpStatsLogger("non_blocking_read");
    OpStatsLogger blockingReadStats = statsLogger.getOpStatsLogger("blocking_read");
    Counter nullReadCounter = statsLogger.getCounter("null_read");
    logger.info("Created dlm for stream {}.", streamName);
    LogReader reader = null;
    Long lastTxId = null;
    while (null == reader) {
        // initialize the last txid
        if (null == lastTxId) {
            switch(readMode) {
                case OLDEST:
                    lastTxId = 0L;
                    break;
                case LATEST:
                    try {
                        lastTxId = dlm.getLastTxId();
                    } catch (IOException ioe) {
                        continue;
                    }
                    break;
                case REWIND:
                    lastTxId = System.currentTimeMillis() - rewindMs;
                    break;
                case POSITION:
                    lastTxId = fromTxId;
                    break;
                default:
                    logger.warn("Unsupported mode {}", readMode);
                    printUsage();
                    System.exit(0);
                    break;
            }
            logger.info("Reading from transaction id {}", lastTxId);
        }
        // Open the reader
        Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            reader = dlm.getInputStream(lastTxId);
            long elapsedMs = stopwatch.elapsed(TimeUnit.MICROSECONDS);
            openReaderStats.registerSuccessfulEvent(elapsedMs);
            logger.info("It took {} ms to position the reader to transaction id {}", lastTxId);
        } catch (IOException ioe) {
            openReaderStats.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
            logger.warn("Failed to create reader for stream {} reading from {}.", streamName, lastTxId);
        }
        if (null == reader) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
            continue;
        }
        // read loop
        LogRecord record;
        boolean nonBlocking = false;
        stopwatch = Stopwatch.createUnstarted();
        while (true) {
            try {
                stopwatch.start();
                record = reader.readNext(nonBlocking);
                if (null != record) {
                    long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                    if (nonBlocking) {
                        nonBlockingReadStats.registerSuccessfulEvent(elapsedMicros);
                    } else {
                        blockingReadStats.registerSuccessfulEvent(elapsedMicros);
                    }
                    lastTxId = record.getTransactionId();
                } else {
                    nullReadCounter.inc();
                }
                if (null == record && !nonBlocking) {
                    nonBlocking = true;
                }
                stopwatch.reset();
            } catch (IOException e) {
                logger.warn("Encountered reading record from stream {} : ", streamName, e);
                reader = null;
                break;
            }
        }
        try {
            TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
        } catch (InterruptedException e) {
        }
    }
}

Example 17 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class DLWriterWorker method close.

@Override
public void close() throws IOException {
    this.running = false;
    SchedulerUtils.shutdownScheduler(this.executorService, 2, TimeUnit.MINUTES);
    SchedulerUtils.shutdownScheduler(this.rescueService, 2, TimeUnit.MINUTES);
    for (AsyncLogWriter writer : streamWriters) {
        FutureUtils.result(writer.asyncClose());
    }
    for (DistributedLogManager dlm : dlms) {
        dlm.close();
    }
    namespace.close();
}

Example 18 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogServer method testHeartbeat.

@Test(timeout = 60000)
public void testHeartbeat() throws Exception {
    String name = "dlserver-heartbeat";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    for (long i = 1; i <= 10; i++) {
        logger.debug("Send heartbeat {} to stream {}.", i, name);
        dlClient.dlClient.check(name).get();
    }
    logger.debug("Write entry one to stream {}.", name);
    dlClient.dlClient.write(name, ByteBuffer.wrap("1".getBytes())).get();
    Thread.sleep(1000);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader = dlm.getInputStream(DLSN.InitialDLSN);
    int numRead = 0;
    // eid=0 => control records
    // other 9 heartbeats will not trigger writing any control records.
    // eid=1 => user entry
    long startEntryId = 1;
    LogRecordWithDLSN r = reader.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(numRead + 1, i);
        assertEquals(r.getDlsn().compareTo(new DLSN(1, startEntryId, 0)), 0);
        ++numRead;
        ++startEntryId;
        r = reader.readNext(false);
    }
    assertEquals(1, numRead);
}

Example 19 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogServer method runSimpleBulkWriteTest.

private void runSimpleBulkWriteTest(int writeCount) throws Exception {
    String name = String.format("dlserver-bulk-write-%d", writeCount);
    dlClient.routingService.addHost(name, dlServer.getAddress());
    List<ByteBuffer> writes = new ArrayList<ByteBuffer>(writeCount);
    for (long i = 1; i <= writeCount; i++) {
        writes.add(ByteBuffer.wrap(("" + i).getBytes()));
    }
    logger.debug("Write {} entries to stream {}.", writeCount, name);
    List<Future<DLSN>> futures = dlClient.dlClient.writeBulk(name, writes);
    assertEquals(futures.size(), writeCount);
    for (Future<DLSN> future : futures) {
        // No throw == pass.
        DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
    }
    DistributedLogManager dlm = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader = dlm.getInputStream(1);
    int numRead = 0;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(numRead + 1, i);
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(writeCount, numRead);
    reader.close();
    dlm.close();
}

Example 20 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogTool method testToolCreateZkAclId.

@Test(timeout = 60000)
public void testToolCreateZkAclId() throws Exception {
    createStream(defaultUri, "0", "CreateAclStream", defaultPrivilegedZkAclId);
    try {
        DistributedLogManager dlm = DLMTestUtil.createNewDLM("0CreateAclStream", conf, defaultUri);
        DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 1000);
        dlm.close();
    } catch (ZKException ex) {
        assertEquals(KeeperException.Code.NOAUTH, ex.getKeeperExceptionCode());
    }
}