Examples with DistributedLogNamespace com.twitter.distributedlog.namespace.DistributedLogNamespace used on opensource projects

Example 11 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestBKDistributedLogManager method testInvalidStreamFromInvalidZkPath.

@Test(timeout = 60000)
public void testInvalidStreamFromInvalidZkPath() throws Exception {
    String baseName = testNames.getMethodName();
    String streamName = "\0blah";
    URI uri = createDLMURI("/" + baseName);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    DistributedLogManager dlm = null;
    AsyncLogWriter writer = null;
    try {
        dlm = namespace.openLog(streamName);
        writer = dlm.startAsyncLogSegmentNonPartitioned();
        fail("should have thrown");
    } catch (InvalidStreamNameException e) {
    } finally {
        if (null != writer) {
            Utils.close(writer);
        }
        if (null != dlm) {
            dlm.close();
        }
        namespace.close();
    }
}

Example 12 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestBKDistributedLogNamespace method testNamespaceListener.

@Test(timeout = 60000)
public void testNamespaceListener() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    final CountDownLatch[] latches = new CountDownLatch[3];
    for (int i = 0; i < 3; i++) {
        latches[i] = new CountDownLatch(1);
    }
    final AtomicInteger numUpdates = new AtomicInteger(0);
    final AtomicInteger numFailures = new AtomicInteger(0);
    final AtomicReference<Collection<String>> receivedStreams = new AtomicReference<Collection<String>>(null);
    namespace.registerNamespaceListener(new NamespaceListener() {

        @Override
        public void onStreamsChanged(Iterator<String> streams) {
            Set<String> streamSet = Sets.newHashSet(streams);
            int updates = numUpdates.incrementAndGet();
            if (streamSet.size() != updates - 1) {
                numFailures.incrementAndGet();
            }
            receivedStreams.set(streamSet);
            latches[updates - 1].countDown();
        }
    });
    latches[0].await();
    BKDistributedLogManager.createLog(conf, zooKeeperClient, uri, "test1");
    latches[1].await();
    BKDistributedLogManager.createLog(conf, zooKeeperClient, uri, "test2");
    latches[2].await();
    assertEquals(0, numFailures.get());
    assertNotNull(receivedStreams.get());
    Set<String> streamSet = new HashSet<String>();
    streamSet.addAll(receivedStreams.get());
    assertEquals(2, receivedStreams.get().size());
    assertEquals(2, streamSet.size());
    assertTrue(streamSet.contains("test1"));
    assertTrue(streamSet.contains("test2"));
}

Example 13 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestBKDistributedLogNamespace method initDlogMeta.

private void initDlogMeta(String dlNamespace, String un, String streamName) throws Exception {
    URI uri = createDLMURI(dlNamespace);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(true);
    newConf.setZkAclId(un);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(streamName);
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (int i = 0; i < 10; i++) {
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
    }
    writer.close();
    dlm.close();
    namespace.close();
}

Example 14 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestAsyncReaderLock method testReaderLockMultiReadersScenario.

@Test(timeout = 60000)
public void testReaderLockMultiReadersScenario() throws Exception {
    final String name = runtime.getMethodName();
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    // Force immediate flush to make dlsn counting easy.
    DistributedLogConfiguration localConf = new DistributedLogConfiguration();
    localConf.addConfiguration(conf);
    localConf.setImmediateFlushEnabled(true);
    localConf.setOutputBufferSize(0);
    // Otherwise, we won't be able to run scheduled threads for readahead when we're in a callback.
    localConf.setNumWorkerThreads(2);
    localConf.setLockTimeout(Long.MAX_VALUE);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId("main").build();
    DistributedLogManager dlm0 = namespace.openLog(name);
    DLMTestUtil.generateCompletedLogSegments(dlm0, localConf, 9, 100);
    dlm0.close();
    int recordCount = 0;
    AtomicReference<DLSN> currentDLSN = new AtomicReference<DLSN>(DLSN.InitialDLSN);
    String clientId1 = "reader1";
    DistributedLogNamespace namespace1 = DistributedLogNamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId1).build();
    DistributedLogManager dlm1 = namespace1.openLog(name);
    String clientId2 = "reader2";
    DistributedLogNamespace namespace2 = DistributedLogNamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId2).build();
    DistributedLogManager dlm2 = namespace2.openLog(name);
    String clientId3 = "reader3";
    DistributedLogNamespace namespace3 = DistributedLogNamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId3).build();
    DistributedLogManager dlm3 = namespace3.openLog(name);
    LOG.info("{} is opening reader on stream {}", clientId1, name);
    Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    AsyncLogReader reader1 = Await.result(futureReader1);
    LOG.info("{} opened reader on stream {}", clientId1, name);
    LOG.info("{} is opening reader on stream {}", clientId2, name);
    Future<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    LOG.info("{} is opening reader on stream {}", clientId3, name);
    Future<AsyncLogReader> futureReader3 = dlm3.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    ExecutorService executorService = Executors.newCachedThreadPool();
    ReadRecordsListener listener2 = new ReadRecordsListener(currentDLSN, clientId2, executorService);
    ReadRecordsListener listener3 = new ReadRecordsListener(currentDLSN, clientId3, executorService);
    futureReader2.addEventListener(listener2);
    futureReader3.addEventListener(listener3);
    // Get reader1 and start reading.
    for (; recordCount < 200; recordCount++) {
        LogRecordWithDLSN record = Await.result(reader1.readNext());
        currentDLSN.set(record.getDlsn());
    }
    // Take a break, reader2 decides to stop waiting and cancels.
    Thread.sleep(1000);
    assertFalse(listener2.done());
    FutureUtils.cancel(futureReader2);
    listener2.getLatch().await();
    assertTrue(listener2.done());
    assertTrue(listener2.failed());
    // Reader1 starts reading again.
    for (; recordCount < 300; recordCount++) {
        LogRecordWithDLSN record = Await.result(reader1.readNext());
        currentDLSN.set(record.getDlsn());
    }
    // Reader1 is done, someone else can take over. Since reader2 was
    // aborted, reader3 should take its place.
    assertFalse(listener3.done());
    Utils.close(reader1);
    listener3.getLatch().await();
    assertTrue(listener3.done());
    assertFalse(listener3.failed());
    assertEquals(new DLSN(3, 99, 0), currentDLSN.get());
    try {
        Await.result(futureReader2);
    } catch (Exception ex) {
    // Can't get this one to close it--the dlm will take care of it.
    }
    Utils.close(Await.result(futureReader3));
    dlm1.close();
    dlm2.close();
    dlm3.close();
    executorService.shutdown();
}

Example 15 with DistributedLogNamespace

use of com.twitter.distributedlog.namespace.DistributedLogNamespace in project distributedlog by twitter.

the class TestAsyncReaderWriter method testSimpleAsyncReadWriteStartEmptyFactory.

/**
     * Test Case: starting reading when the streams don't exist.
     * {@link https://issues.apache.org/jira/browse/DL-42}
     */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 120000)
public void testSimpleAsyncReadWriteStartEmptyFactory() throws Exception {
    int count = 50;
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    confLocal.setOutputBufferSize(1024);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 1;
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    final DistributedLogManager[] dlms = new DistributedLogManager[count];
    final TestReader[] readers = new TestReader[count];
    final CountDownLatch readyLatch = new CountDownLatch(count);
    final CountDownLatch[] syncLatches = new CountDownLatch[count];
    final CountDownLatch[] readerDoneLatches = new CountDownLatch[count];
    for (int s = 0; s < count; s++) {
        dlms[s] = namespace.openLog(name + String.format("%d", s));
        readerDoneLatches[s] = new CountDownLatch(1);
        syncLatches[s] = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
        readers[s] = new TestReader("reader-" + s, dlms[s], DLSN.InitialDLSN, false, 0, readyLatch, syncLatches[s], readerDoneLatches[s]);
        readers[s].start();
    }
    // wait all readers were positioned at least once
    readyLatch.await();
    final CountDownLatch writeLatch = new CountDownLatch(3 * count);
    final AtomicBoolean writeErrors = new AtomicBoolean(false);
    int txid = 1;
    for (long i = 0; i < 3; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter[] writers = new BKAsyncLogWriter[count];
        for (int s = 0; s < count; s++) {
            writers[s] = (BKAsyncLogWriter) (dlms[s].startAsyncLogSegmentNonPartitioned());
        }
        for (long j = 0; j < 1; j++) {
            final long currentEntryId = j;
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            for (int s = 0; s < count; s++) {
                Future<DLSN> dlsnFuture = writers[s].write(record);
                dlsnFuture.addEventListener(new WriteFutureEventListener(record, currentLogSegmentSeqNo, currentEntryId, writeLatch, writeErrors, true));
            }
        }
        for (int s = 0; s < count; s++) {
            writers[s].closeAndComplete();
        }
    }
    writeLatch.await();
    assertFalse("All writes should succeed", writeErrors.get());
    for (int s = 0; s < count; s++) {
        readerDoneLatches[s].await();
        assertFalse("Reader " + s + " should not encounter errors", readers[s].areErrorsFound());
        syncLatches[s].await();
        assertEquals(numLogSegments * numRecordsPerLogSegment, readers[s].getNumReads().get());
        assertTrue("Reader " + s + " should position at least once", readers[s].getNumReaderPositions().get() > 0);
    }
    for (int s = 0; s < count; s++) {
        readers[s].stop();
        dlms[s].close();
    }
}