Examples with DistributedLogConfiguration com.twitter.distributedlog.DistributedLogConfiguration used on opensource projects

Example 6 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    BookKeeperClient bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    bkc = getBookKeeperClient(factory);
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), executorService, bkc, confLocal.getBKDigestPW(), false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    factory.close();
}

Example 7 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
     * {@link https://issues.apache.org/jira/browse/DL-44}
     */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    com.twitter.distributedlog.DistributedLogManagerFactory factory = new com.twitter.distributedlog.DistributedLogManagerFactory(confLocal, uri);
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = factory.createDistributedLogManagerWithSharedClients(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecord record = Await.result(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    DLSN dlsn = readDLM.getLastDLSN();
    assertTrue(dlsn.compareTo(new DLSN(5, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(4, -1, Long.MIN_VALUE)) > 0);
    // there isn't records should be read
    Future<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        Await.result(readFuture, Duration.fromMilliseconds(1000));
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (TimeoutException te) {
    // expected
    }
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(factory, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(factory)), streamName, false, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    dlsn = readDLM.getLastDLSN();
    assertTrue(dlsn.compareTo(new DLSN(5, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(4, -1, Long.MIN_VALUE)) > 0);
    // there isn't records should be read
    try {
        Await.result(readFuture, Duration.fromMilliseconds(1000));
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (TimeoutException te) {
    // expected
    }
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(factory, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(factory)), streamName, false, false);
    // Wait for reader to be aware of new log segments
    TimeUnit.SECONDS.sleep(2);
    expectedTxId = 51L;
    LogRecord record = Await.result(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Await.result(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    dlsn = readDLM.getLastDLSN();
    LOG.info("LastDLSN after fix inprogress segment : {}", dlsn);
    assertTrue(dlsn.compareTo(new DLSN(7, Long.MIN_VALUE, Long.MIN_VALUE)) < 0);
    assertTrue(dlsn.compareTo(new DLSN(6, -1, Long.MIN_VALUE)) > 0);
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    factory.close();
}

Example 8 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestLedgerAllocator method testAllocatorWithoutEnoughBookies.

@Test(timeout = 60000)
public void testAllocatorWithoutEnoughBookies() throws Exception {
    String allocationPath = "/allocator-without-enough-bookies";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setEnsembleSize(numBookies * 2);
    confLocal.setWriteQuorumSize(numBookies * 2);
    SimpleLedgerAllocator allocator1 = createAllocator(allocationPath, confLocal);
    allocator1.allocate();
    ZKTransaction txn1 = newTxn();
    try {
        FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER));
        fail("Should fail allocating ledger if there aren't enough bookies");
    } catch (AllocationException ioe) {
        // expected
        assertEquals(Phase.ERROR, ioe.getPhase());
    }
    byte[] data = zkc.get().getData(allocationPath, false, null);
    assertEquals(0, data.length);
}

Example 9 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestZKNamespaceWatcher method testSessionExpired.

@Test(timeout = 60000)
public void testSessionExpired() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.addConfiguration(baseConf);
    ZKNamespaceWatcher watcher = new ZKNamespaceWatcher(conf, uri, zkc, scheduler);
    final CountDownLatch[] latches = new CountDownLatch[10];
    for (int i = 0; i < 10; i++) {
        latches[i] = new CountDownLatch(1);
    }
    final AtomicInteger numUpdates = new AtomicInteger(0);
    final AtomicReference<Set<String>> receivedLogs = new AtomicReference<Set<String>>(null);
    watcher.registerListener(new NamespaceListener() {

        @Override
        public void onStreamsChanged(Iterator<String> streams) {
            Set<String> streamSet = Sets.newHashSet(streams);
            int updates = numUpdates.incrementAndGet();
            receivedLogs.set(streamSet);
            latches[updates - 1].countDown();
        }
    });
    latches[0].await();
    createLogInNamespace(uri, "test1");
    latches[1].await();
    createLogInNamespace(uri, "test2");
    latches[2].await();
    assertEquals(2, receivedLogs.get().size());
    ZooKeeperClientUtils.expireSession(zkc, DLUtils.getZKServersFromDLUri(uri), zkSessionTimeoutMs);
    latches[3].await();
    assertEquals(2, receivedLogs.get().size());
    createLogInNamespace(uri, "test3");
    latches[4].await();
    assertEquals(3, receivedLogs.get().size());
}

Example 10 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestFederatedZKLogMetadataStore method testCreateLog.

@Test(timeout = 60000)
public void testCreateLog() throws Exception {
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.addConfiguration(baseConf);
    ZooKeeperClient anotherZkc = TestZooKeeperClientBuilder.newBuilder().uri(uri).sessionTimeoutMs(zkSessionTimeoutMs).build();
    FederatedZKLogMetadataStore anotherMetadataStore = new FederatedZKLogMetadataStore(conf, uri, anotherZkc, scheduler);
    for (int i = 0; i < 2 * maxLogsPerSubnamespace; i++) {
        LogMetadataStore createStore, checkStore;
        if (i % 2 == 0) {
            createStore = metadataStore;
            checkStore = anotherMetadataStore;
        } else {
            createStore = anotherMetadataStore;
            checkStore = metadataStore;
        }
        String logName = "test-create-log-" + i;
        URI logUri = FutureUtils.result(createStore.createLog(logName));
        Optional<URI> logLocation = FutureUtils.result(checkStore.getLogLocation(logName));
        assertTrue("Log " + logName + " doesn't exist", logLocation.isPresent());
        assertEquals("Different log location " + logLocation.get() + " is found", logUri, logLocation.get());
    }
    assertEquals(2, metadataStore.getSubnamespaces().size());
    assertEquals(2, anotherMetadataStore.getSubnamespaces().size());
}