Examples with DLSN com.twitter.distributedlog.DLSN used on opensource projects

Example 1 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class AsyncReaderBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    logger.info("Created dlm for stream {}.", streamName);
    // Stats
    OpStatsLogger openReaderStats = statsLogger.getOpStatsLogger("open_reader");
    OpStatsLogger blockingReadStats = statsLogger.getOpStatsLogger("blocking_read");
    Counter readCounter = statsLogger.getCounter("reads");
    AsyncLogReader reader = null;
    DLSN lastDLSN = null;
    Long lastTxId = null;
    while (null == reader) {
        // initialize the last txid
        if (null == lastTxId) {
            switch(readMode) {
                case OLDEST:
                    lastTxId = 0L;
                    lastDLSN = DLSN.InitialDLSN;
                    break;
                case LATEST:
                    lastTxId = Long.MAX_VALUE;
                    try {
                        lastDLSN = dlm.getLastDLSN();
                    } catch (IOException ioe) {
                        continue;
                    }
                    break;
                case REWIND:
                    lastTxId = System.currentTimeMillis() - rewindMs;
                    lastDLSN = null;
                    break;
                case POSITION:
                    lastTxId = fromTxId;
                    lastDLSN = null;
                    break;
                default:
                    logger.warn("Unsupported mode {}", readMode);
                    printUsage();
                    System.exit(0);
                    break;
            }
            logger.info("Reading from transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        }
        // Open the reader
        Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            if (null == lastDLSN) {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastTxId));
            } else {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastDLSN));
            }
            long elapsedMs = stopwatch.elapsed(TimeUnit.MICROSECONDS);
            openReaderStats.registerSuccessfulEvent(elapsedMs);
            logger.info("It took {} ms to position the reader to transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        } catch (IOException ioe) {
            openReaderStats.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
            logger.warn("Failed to create reader for stream {} reading from tx id = {}, dlsn = {}.", new Object[] { streamName, lastTxId, lastDLSN });
        }
        if (null == reader) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
            continue;
        }
        List<LogRecordWithDLSN> records;
        stopwatch = Stopwatch.createUnstarted();
        while (true) {
            try {
                stopwatch.start();
                records = FutureUtils.result(reader.readBulk(batchSize));
                long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                blockingReadStats.registerSuccessfulEvent(elapsedMicros);
                if (!records.isEmpty()) {
                    readCounter.add(records.size());
                    LogRecordWithDLSN lastRecord = records.get(records.size() - 1);
                    lastTxId = lastRecord.getTransactionId();
                    lastDLSN = lastRecord.getDlsn();
                }
                stopwatch.reset();
            } catch (IOException e) {
                logger.warn("Encountered reading record from stream {} : ", streamName, e);
                reader = null;
                break;
            }
        }
        try {
            TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
        } catch (InterruptedException e) {
        }
    }
}

Example 2 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class LogSegmentMetadataStoreUpdater method updateLastRecord.

@Override
public Future<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment, LogRecordWithDLSN record) {
    DLSN dlsn = record.getDlsn();
    Preconditions.checkState(!segment.isInProgress(), "Updating last dlsn for an inprogress log segment isn't supported.");
    Preconditions.checkArgument(segment.isDLSNinThisSegment(dlsn), "DLSN " + dlsn + " doesn't belong to segment " + segment);
    final LogSegmentMetadata newSegment = segment.mutator().setLastDLSN(dlsn).setLastTxId(record.getTransactionId()).setRecordCount(record).build();
    return updateSegmentMetadata(newSegment);
}

Example 3 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogServer method testTruncateStream.

@Test(timeout = 60000)
public void testTruncateStream() throws Exception {
    String name = "dlserver-truncate-stream";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    long txid = 1;
    Map<Long, DLSN> txid2DLSN = new HashMap<Long, DLSN>();
    for (int s = 1; s <= 3; s++) {
        for (long i = 1; i <= 10; i++) {
            long curTxId = txid++;
            logger.debug("Write entry {} to stream {}.", curTxId, name);
            DLSN dlsn = dlClient.dlClient.write(name, ByteBuffer.wrap(("" + curTxId).getBytes())).get();
            txid2DLSN.put(curTxId, dlsn);
        }
        if (s <= 2) {
            dlClient.dlClient.release(name).get();
        }
    }
    DLSN dlsnToDelete = txid2DLSN.get(21L);
    dlClient.dlClient.truncate(name, dlsnToDelete).get();
    DistributedLogManager readDLM = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader = readDLM.getInputStream(1);
    int numRead = 0;
    int curTxId = 11;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(curTxId++, i);
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(20, numRead);
    reader.close();
    readDLM.close();
}

Example 4 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogServer method testDeleteStream.

@Test(timeout = 60000)
public void testDeleteStream() throws Exception {
    String name = "dlserver-delete-stream";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    long txid = 101;
    for (long i = 1; i <= 10; i++) {
        long curTxId = txid++;
        logger.debug("Write entry {} to stream {}.", curTxId, name);
        dlClient.dlClient.write(name, ByteBuffer.wrap(("" + curTxId).getBytes())).get();
    }
    checkStream(1, 1, 1, name, dlServer.getAddress(), true, true);
    dlClient.dlClient.delete(name).get();
    checkStream(0, 0, 0, name, dlServer.getAddress(), false, false);
    Thread.sleep(1000);
    DistributedLogManager dlm101 = DLMTestUtil.createNewDLM(name, conf, getUri());
    AsyncLogReader reader101 = FutureUtils.result(dlm101.openAsyncLogReader(DLSN.InitialDLSN));
    try {
        FutureUtils.result(reader101.readNext());
        fail("Should fail with LogNotFoundException since the stream is deleted");
    } catch (LogNotFoundException lnfe) {
    // expected
    }
    FutureUtils.result(reader101.asyncClose());
    dlm101.close();
    txid = 201;
    for (long i = 1; i <= 10; i++) {
        long curTxId = txid++;
        logger.debug("Write entry {} to stream {}.", curTxId, name);
        DLSN dlsn = dlClient.dlClient.write(name, ByteBuffer.wrap(("" + curTxId).getBytes())).get();
    }
    Thread.sleep(1000);
    DistributedLogManager dlm201 = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader201 = dlm201.getInputStream(1);
    int numRead = 0;
    int curTxId = 201;
    LogRecord r = reader201.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(curTxId++, i);
        ++numRead;
        r = reader201.readNext(false);
    }
    assertEquals(10, numRead);
    reader201.close();
    dlm201.close();
}

Example 5 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogServer method testBulkWritePartialFailure.

@Test(timeout = 60000)
public void testBulkWritePartialFailure() throws Exception {
    String name = String.format("dlserver-bulk-write-%s", "partial-failure");
    dlClient.routingService.addHost(name, dlServer.getAddress());
    final int writeCount = 100;
    List<ByteBuffer> writes = new ArrayList<ByteBuffer>(writeCount * 2 + 1);
    for (long i = 1; i <= writeCount; i++) {
        writes.add(ByteBuffer.wrap(("" + i).getBytes()));
    }
    // Too big, will cause partial failure.
    ByteBuffer buf = ByteBuffer.allocate(MAX_LOGRECORD_SIZE + 1);
    writes.add(buf);
    for (long i = 1; i <= writeCount; i++) {
        writes.add(ByteBuffer.wrap(("" + i).getBytes()));
    }
    // Count succeeded.
    List<Future<DLSN>> futures = dlClient.dlClient.writeBulk(name, writes);
    int succeeded = 0;
    for (int i = 0; i < writeCount; i++) {
        Future<DLSN> future = futures.get(i);
        try {
            DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
            ++succeeded;
        } catch (Exception ex) {
            failDueToWrongException(ex);
        }
    }
    validateFailedAsLogRecordTooLong(futures.get(writeCount));
    FutureUtils.result(Futures.collect(futures.subList(writeCount + 1, 2 * writeCount + 1)));
    assertEquals(writeCount, succeeded);
}