Examples with DistributedLogManager com.twitter.distributedlog.DistributedLogManager used on opensource projects

Example 1 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogServer method testTruncateStream.

@Test(timeout = 60000)
public void testTruncateStream() throws Exception {
    String name = "dlserver-truncate-stream";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    long txid = 1;
    Map<Long, DLSN> txid2DLSN = new HashMap<Long, DLSN>();
    for (int s = 1; s <= 3; s++) {
        for (long i = 1; i <= 10; i++) {
            long curTxId = txid++;
            logger.debug("Write entry {} to stream {}.", curTxId, name);
            DLSN dlsn = dlClient.dlClient.write(name, ByteBuffer.wrap(("" + curTxId).getBytes())).get();
            txid2DLSN.put(curTxId, dlsn);
        }
        if (s <= 2) {
            dlClient.dlClient.release(name).get();
        }
    }
    DLSN dlsnToDelete = txid2DLSN.get(21L);
    dlClient.dlClient.truncate(name, dlsnToDelete).get();
    DistributedLogManager readDLM = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader = readDLM.getInputStream(1);
    int numRead = 0;
    int curTxId = 11;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(curTxId++, i);
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(20, numRead);
    reader.close();
    readDLM.close();
}

Example 2 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogServer method testBasicWrite.

/**
 * {@link https://issues.apache.org/jira/browse/DL-27}
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
public void testBasicWrite() throws Exception {
    String name = "dlserver-basic-write";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    for (long i = 1; i <= 10; i++) {
        logger.debug("Write entry {} to stream {}.", i, name);
        Await.result(dlClient.dlClient.write(name, ByteBuffer.wrap(("" + i).getBytes())));
    }
    HeartbeatOptions hbOptions = new HeartbeatOptions();
    hbOptions.setSendHeartBeatToReader(true);
    // make sure the first log segment of each stream created
    FutureUtils.result(dlClient.dlClient.heartbeat(name));
    DistributedLogManager dlm = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader = dlm.getInputStream(1);
    int numRead = 0;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(numRead + 1, i);
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(10, numRead);
    reader.close();
    dlm.close();
}

Example 3 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogServer method testDeleteStream.

@Test(timeout = 60000)
public void testDeleteStream() throws Exception {
    String name = "dlserver-delete-stream";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    long txid = 101;
    for (long i = 1; i <= 10; i++) {
        long curTxId = txid++;
        logger.debug("Write entry {} to stream {}.", curTxId, name);
        dlClient.dlClient.write(name, ByteBuffer.wrap(("" + curTxId).getBytes())).get();
    }
    checkStream(1, 1, 1, name, dlServer.getAddress(), true, true);
    dlClient.dlClient.delete(name).get();
    checkStream(0, 0, 0, name, dlServer.getAddress(), false, false);
    Thread.sleep(1000);
    DistributedLogManager dlm101 = DLMTestUtil.createNewDLM(name, conf, getUri());
    AsyncLogReader reader101 = FutureUtils.result(dlm101.openAsyncLogReader(DLSN.InitialDLSN));
    try {
        FutureUtils.result(reader101.readNext());
        fail("Should fail with LogNotFoundException since the stream is deleted");
    } catch (LogNotFoundException lnfe) {
    // expected
    }
    FutureUtils.result(reader101.asyncClose());
    dlm101.close();
    txid = 201;
    for (long i = 1; i <= 10; i++) {
        long curTxId = txid++;
        logger.debug("Write entry {} to stream {}.", curTxId, name);
        DLSN dlsn = dlClient.dlClient.write(name, ByteBuffer.wrap(("" + curTxId).getBytes())).get();
    }
    Thread.sleep(1000);
    DistributedLogManager dlm201 = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader201 = dlm201.getInputStream(1);
    int numRead = 0;
    int curTxId = 201;
    LogRecord r = reader201.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(curTxId++, i);
        ++numRead;
        r = reader201.readNext(false);
    }
    assertEquals(10, numRead);
    reader201.close();
    dlm201.close();
}

Example 4 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDistributedLogServer method testFenceWrite.

@Test(timeout = 60000)
public void testFenceWrite() throws Exception {
    String name = "dlserver-fence-write";
    dlClient.routingService.addHost(name, dlServer.getAddress());
    for (long i = 1; i <= 10; i++) {
        logger.debug("Write entry {} to stream {}.", i, name);
        dlClient.dlClient.write(name, ByteBuffer.wrap(("" + i).getBytes())).get();
    }
    Thread.sleep(1000);
    logger.info("Fencing stream {}.", name);
    DLMTestUtil.fenceStream(conf, getUri(), name);
    logger.info("Fenced stream {}.", name);
    for (long i = 11; i <= 20; i++) {
        logger.debug("Write entry {} to stream {}.", i, name);
        dlClient.dlClient.write(name, ByteBuffer.wrap(("" + i).getBytes())).get();
    }
    DistributedLogManager dlm = DLMTestUtil.createNewDLM(name, conf, getUri());
    LogReader reader = dlm.getInputStream(1);
    int numRead = 0;
    LogRecord r = reader.readNext(false);
    while (null != r) {
        int i = Integer.parseInt(new String(r.getPayload()));
        assertEquals(numRead + 1, i);
        ++numRead;
        r = reader.readNext(false);
    }
    assertEquals(20, numRead);
    reader.close();
    dlm.close();
}

Example 5 with DistributedLogManager

use of com.twitter.distributedlog.DistributedLogManager in project distributedlog by twitter.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    BookKeeperClient bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    factory.close();
}