Examples with LogRecordWithDLSN com.twitter.distributedlog.LogRecordWithDLSN used on opensource projects

Example 1 with LogRecordWithDLSN

use of com.twitter.distributedlog.LogRecordWithDLSN in project distributedlog by twitter.

the class LogSegmentMetadataStoreUpdater method updateLastRecord.

@Override
public Future<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment, LogRecordWithDLSN record) {
    DLSN dlsn = record.getDlsn();
    Preconditions.checkState(!segment.isInProgress(), "Updating last dlsn for an inprogress log segment isn't supported.");
    Preconditions.checkArgument(segment.isDLSNinThisSegment(dlsn), "DLSN " + dlsn + " doesn't belong to segment " + segment);
    final LogSegmentMetadata newSegment = segment.mutator().setLastDLSN(dlsn).setLastTxId(record.getTransactionId()).setRecordCount(record).build();
    return updateSegmentMetadata(newSegment);
}

Example 2 with LogRecordWithDLSN

use of com.twitter.distributedlog.LogRecordWithDLSN in project distributedlog by twitter.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
 * {@link https://issues.apache.org/jira/browse/DL-44}
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = factory.createDistributedLogManagerWithSharedClients(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecord record = Await.result(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    DLSN dlsn = readDLM.getLastDLSN();
    assertTrue(dlsn.compareTo(new DLSN(5, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(4, -1, Long.MIN_VALUE)) > 0);
    // there isn't records should be read
    Future<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        Await.result(readFuture, Duration.fromMilliseconds(1000));
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (TimeoutException te) {
    // expected
    }
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), streamName, false, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    dlsn = readDLM.getLastDLSN();
    assertTrue(dlsn.compareTo(new DLSN(5, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(4, -1, Long.MIN_VALUE)) > 0);
    // there isn't records should be read
    try {
        Await.result(readFuture, Duration.fromMilliseconds(1000));
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (TimeoutException te) {
    // expected
    }
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), streamName, false, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    expectedTxId = 51L;
    LogRecord record = Await.result(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Await.result(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    dlsn = readDLM.getLastDLSN();
    LOG.info("LastDLSN after fix inprogress segment : {}", dlsn);
    assertTrue(dlsn.compareTo(new DLSN(7, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(6, -1, Long.MIN_VALUE)) > 0);
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    factory.close();
}

Example 3 with LogRecordWithDLSN

use of com.twitter.distributedlog.LogRecordWithDLSN in project distributedlog by twitter.

the class AsyncReaderBenchmark method benchmark.

@Override
protected void benchmark(DistributedLogNamespace namespace, String logName, StatsLogger statsLogger) {
    DistributedLogManager dlm = null;
    while (null == dlm) {
        try {
            dlm = namespace.openLog(streamName);
        } catch (IOException ioe) {
            logger.warn("Failed to create dlm for stream {} : ", streamName, ioe);
        }
        if (null == dlm) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
        }
    }
    logger.info("Created dlm for stream {}.", streamName);
    // Stats
    OpStatsLogger openReaderStats = statsLogger.getOpStatsLogger("open_reader");
    OpStatsLogger blockingReadStats = statsLogger.getOpStatsLogger("blocking_read");
    Counter readCounter = statsLogger.getCounter("reads");
    AsyncLogReader reader = null;
    DLSN lastDLSN = null;
    Long lastTxId = null;
    while (null == reader) {
        // initialize the last txid
        if (null == lastTxId) {
            switch(readMode) {
                case OLDEST:
                    lastTxId = 0L;
                    lastDLSN = DLSN.InitialDLSN;
                    break;
                case LATEST:
                    lastTxId = Long.MAX_VALUE;
                    try {
                        lastDLSN = dlm.getLastDLSN();
                    } catch (IOException ioe) {
                        continue;
                    }
                    break;
                case REWIND:
                    lastTxId = System.currentTimeMillis() - rewindMs;
                    lastDLSN = null;
                    break;
                case POSITION:
                    lastTxId = fromTxId;
                    lastDLSN = null;
                    break;
                default:
                    logger.warn("Unsupported mode {}", readMode);
                    printUsage();
                    System.exit(0);
                    break;
            }
            logger.info("Reading from transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        }
        // Open the reader
        Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            if (null == lastDLSN) {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastTxId));
            } else {
                reader = FutureUtils.result(dlm.openAsyncLogReader(lastDLSN));
            }
            long elapsedMs = stopwatch.elapsed(TimeUnit.MICROSECONDS);
            openReaderStats.registerSuccessfulEvent(elapsedMs);
            logger.info("It took {} ms to position the reader to transaction id = {}, dlsn = {}", lastTxId, lastDLSN);
        } catch (IOException ioe) {
            openReaderStats.registerFailedEvent(stopwatch.elapsed(TimeUnit.MICROSECONDS));
            logger.warn("Failed to create reader for stream {} reading from tx id = {}, dlsn = {}.", new Object[] { streamName, lastTxId, lastDLSN });
        }
        if (null == reader) {
            try {
                TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
            } catch (InterruptedException e) {
            }
            continue;
        }
        List<LogRecordWithDLSN> records;
        stopwatch = Stopwatch.createUnstarted();
        while (true) {
            try {
                stopwatch.start();
                records = FutureUtils.result(reader.readBulk(batchSize));
                long elapsedMicros = stopwatch.stop().elapsed(TimeUnit.MICROSECONDS);
                blockingReadStats.registerSuccessfulEvent(elapsedMicros);
                if (!records.isEmpty()) {
                    readCounter.add(records.size());
                    LogRecordWithDLSN lastRecord = records.get(records.size() - 1);
                    lastTxId = lastRecord.getTransactionId();
                    lastDLSN = lastRecord.getDlsn();
                }
                stopwatch.reset();
            } catch (IOException e) {
                logger.warn("Encountered reading record from stream {} : ", streamName, e);
                reader = null;
                break;
            }
        }
        try {
            TimeUnit.MILLISECONDS.sleep(conf.getZKSessionTimeoutMilliseconds());
        } catch (InterruptedException e) {
        }
    }
}

Example 4 with LogRecordWithDLSN

use of com.twitter.distributedlog.LogRecordWithDLSN in project distributedlog by twitter.

the class TestDistributedLogTool method testToolTruncateStream.

@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", conf, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 1000);
    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);
    assertEquals(0, cmd.runCmd());
    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());
    reader.close();
    dlm.close();
}

Example 5 with LogRecordWithDLSN

use of com.twitter.distributedlog.LogRecordWithDLSN in project distributedlog by twitter.

the class TestLogSegmentMetadataStoreUpdater method testUpdateLastDLSN.

@Test(timeout = 60000)
public void testUpdateLastDLSN() throws Exception {
    String ledgerPath = "/testUpdateLastDLSN";
    zkc.get().create(ledgerPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Create 1 completed log segment
    LogSegmentMetadata completedLogSegment = DLMTestUtil.completedLogSegment(ledgerPath, 1L, 0L, 99L, 100, 1L, 99L, 0L);
    completedLogSegment.write(zkc);
    // Create 1 inprogress log segment
    LogSegmentMetadata inprogressLogSegment = DLMTestUtil.inprogressLogSegment(ledgerPath, 2L, 100L, 2L);
    inprogressLogSegment.write(zkc);
    DLSN badLastDLSN = new DLSN(99L, 0L, 0L);
    DLSN goodLastDLSN1 = new DLSN(1L, 100L, 0L);
    DLSN goodLastDLSN2 = new DLSN(2L, 200L, 0L);
    LogRecordWithDLSN badRecord = DLMTestUtil.getLogRecordWithDLSNInstance(badLastDLSN, 100L);
    LogRecordWithDLSN goodRecord1 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN1, 100L);
    LogRecordWithDLSN goodRecord2 = DLMTestUtil.getLogRecordWithDLSNInstance(goodLastDLSN2, 200L);
    // Dryrun
    MetadataUpdater dryrunUpdater = new DryrunLogSegmentMetadataStoreUpdater(conf, metadataStore);
    try {
        FutureUtils.result(dryrunUpdater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        FutureUtils.result(dryrunUpdater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    LogSegmentMetadata updatedCompletedLogSegment = FutureUtils.result(dryrunUpdater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    Map<Long, LogSegmentMetadata> segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    LogSegmentMetadata readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(completedLogSegment, readCompletedLogSegment);
    LogSegmentMetadata readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
    // Fix the last dlsn
    MetadataUpdater updater = LogSegmentMetadataStoreUpdater.createMetadataUpdater(conf, metadataStore);
    try {
        FutureUtils.result(updater.updateLastRecord(completedLogSegment, badRecord));
        fail("Should fail on updating dlsn that in different log segment");
    } catch (IllegalArgumentException iae) {
    // expected
    }
    try {
        FutureUtils.result(updater.updateLastRecord(inprogressLogSegment, goodRecord2));
        fail("Should fail on updating dlsn for an inprogress log segment");
    } catch (IllegalStateException ise) {
    // expected
    }
    updatedCompletedLogSegment = FutureUtils.result(updater.updateLastRecord(completedLogSegment, goodRecord1));
    assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId());
    assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    segmentList = readLogSegments(ledgerPath);
    assertEquals(2, segmentList.size());
    readCompletedLogSegment = segmentList.get(1L);
    assertNotNull(readCompletedLogSegment);
    assertEquals(goodLastDLSN1, readCompletedLogSegment.getLastDLSN());
    assertEquals(goodRecord1.getTransactionId(), readCompletedLogSegment.getLastTxId());
    assertTrue(readCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1));
    assertEquals(updatedCompletedLogSegment, readCompletedLogSegment);
    assertEquals(completedLogSegment.getCompletionTime(), readCompletedLogSegment.getCompletionTime());
    assertEquals(completedLogSegment.getFirstTxId(), readCompletedLogSegment.getFirstTxId());
    assertEquals(completedLogSegment.getLedgerId(), readCompletedLogSegment.getLedgerId());
    assertEquals(completedLogSegment.getLogSegmentSequenceNumber(), readCompletedLogSegment.getLogSegmentSequenceNumber());
    assertEquals(completedLogSegment.getRegionId(), readCompletedLogSegment.getRegionId());
    assertEquals(completedLogSegment.getZkPath(), readCompletedLogSegment.getZkPath());
    assertEquals(completedLogSegment.getZNodeName(), readCompletedLogSegment.getZNodeName());
    readInprogressLogSegment = segmentList.get(2L);
    assertNotNull(readInprogressLogSegment);
    assertEquals(inprogressLogSegment, readInprogressLogSegment);
}