Examples with DLSN com.twitter.distributedlog.DLSN used on opensource projects

Example 6 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogServer method testBulkWriteEmptyBuffer.

@Test(timeout = 60000)
public void testBulkWriteEmptyBuffer() throws Exception {
    String name = String.format("dlserver-bulk-write-%s", "empty");
    dlClient.routingService.addHost(name, dlServer.getAddress());
    List<ByteBuffer> writes = new ArrayList<ByteBuffer>();
    writes.add(ByteBuffer.wrap(("").getBytes()));
    writes.add(ByteBuffer.wrap(("").getBytes()));
    List<Future<DLSN>> futures = dlClient.dlClient.writeBulk(name, writes);
    assertEquals(2, futures.size());
    for (Future<DLSN> future : futures) {
        // No throw == pass
        DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
    }
}

Example 7 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class WriteOp method executeOp.

@Override
protected Future<WriteResponse> executeOp(AsyncLogWriter writer, Sequencer sequencer, Object txnLock) {
    if (!stream.equals(writer.getStreamName())) {
        logger.error("Write: Stream Name Mismatch in the Stream Map {}, {}", stream, writer.getStreamName());
        return Future.exception(new IllegalStateException("The stream mapping is incorrect, fail the request"));
    }
    long txnId;
    Future<DLSN> writeResult;
    synchronized (txnLock) {
        txnId = sequencer.nextId();
        LogRecord record = new LogRecord(txnId, payload);
        if (isRecordSet) {
            record.setRecordSet();
        }
        writeResult = writer.write(record);
    }
    return writeResult.map(new AbstractFunction1<DLSN, WriteResponse>() {

        @Override
        public WriteResponse apply(DLSN value) {
            successRecordCounter.inc();
            return ResponseUtils.writeSuccess().setDlsn(value.serialize(dlsnVersion));
        }
    });
}

Example 8 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestLogRecordSelectors method testFirstRecordSelector.

@Test(timeout = 60000)
public void testFirstRecordSelector() {
    FirstRecordSelector selectorIncludeControlRecord = new FirstRecordSelector(true);
    for (int i = 0; i < 5; i++) {
        selectorIncludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2, 0L), i * 2, true));
        selectorIncludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2 + 1, 0L), i * 2 + 1));
    }
    assertEquals(new DLSN(1L, 0L, 0L), selectorIncludeControlRecord.result().getDlsn());
    FirstRecordSelector selectorExcludeControlRecord = new FirstRecordSelector(false);
    for (int i = 0; i < 5; i++) {
        selectorExcludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2, 0L), i * 2, true));
        selectorExcludeControlRecord.process(DLMTestUtil.getLogRecordWithDLSNInstance(new DLSN(1L, i * 2 + 1, 0L), i * 2 + 1));
    }
    assertEquals(new DLSN(1L, 1L, 0L), selectorExcludeControlRecord.result().getDlsn());
}

Example 9 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDistributedLogTool method testToolTruncateStream.

@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", conf, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 1000);
    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);
    assertEquals(0, cmd.runCmd());
    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());
    reader.close();
    dlm.close();
}

Example 10 with DLSN

use of com.twitter.distributedlog.DLSN in project distributedlog by twitter.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    BookKeeperClient bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    factory.close();
}