Examples with Namespace org.apache.distributedlog.api.namespace.Namespace used on opensource projects

Example 26 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project incubator-heron by apache.

the class DLDownloaderTest method testOpenInputStream.

@Test
public void testOpenInputStream() throws Exception {
    Namespace ns = mock(Namespace.class);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(ns.openLog(anyString())).thenReturn(dlm);
    when(dlm.getInputStream(eq(DLSN.InitialDLSN))).thenReturn(reader);
    InputStream is = DLDownloader.openInputStream(ns, "test-open-inputstream");
    assertTrue(is instanceof DLInputStream);
    is.close();
    verify(dlm, times(1)).close();
    verify(reader, times(1)).close();
}

Example 27 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project incubator-heron by apache.

the class DLDownloaderTest method testDownload.

@Test
public void testDownload() throws Exception {
    String logName = "test-download";
    URI uri = URI.create("distributedlog://127.0.0.1/test/distributedlog/" + logName);
    File tempFile = File.createTempFile("test", "download");
    // make sure it is deleted when the test completes
    tempFile.deleteOnExit();
    Path path = Paths.get(tempFile.toURI());
    Namespace ns = mock(Namespace.class);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(ns.openLog(anyString())).thenReturn(dlm);
    when(dlm.getInputStream(eq(DLSN.InitialDLSN))).thenReturn(reader);
    when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));
    NamespaceBuilder nsBuilder = mock(NamespaceBuilder.class);
    when(nsBuilder.clientId(anyString())).thenReturn(nsBuilder);
    when(nsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(nsBuilder);
    when(nsBuilder.uri(any(URI.class))).thenReturn(nsBuilder);
    when(nsBuilder.build()).thenReturn(ns);
    PowerMockito.mockStatic(Extractor.class);
    PowerMockito.doNothing().when(Extractor.class, "extract", any(InputStream.class), any(Path.class));
    DLDownloader downloader = new DLDownloader(() -> nsBuilder);
    downloader.download(uri, path);
    URI parentUri = URI.create("distributedlog://127.0.0.1/test/distributedlog");
    verify(nsBuilder, times(1)).clientId(eq("heron-downloader"));
    verify(nsBuilder, times(1)).conf(eq(CONF));
    verify(nsBuilder, times(1)).uri(parentUri);
    PowerMockito.verifyStatic(times(1));
    Extractor.extract(any(InputStream.class), eq(path));
    verify(ns, times(1)).openLog(eq(logName));
    verify(ns, times(1)).close();
}

Example 28 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestAsyncReaderWriter method testAsyncWriteWithMinDelayBetweenFlushesFlushFailure.

@Test(timeout = 60000)
public void testAsyncWriteWithMinDelayBetweenFlushesFlushFailure() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setMinDelayBetweenImmediateFlushMs(1);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).clientId("gabbagoo").build();
    DistributedLogManager dlm = namespace.openLog(name);
    Namespace namespace1 = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).clientId("tortellini").build();
    DistributedLogManager dlm1 = namespace1.openLog(name);
    int txid = 1;
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    // First write succeeds since lock isnt checked until transmit, which is scheduled
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
    writer.flushAndCommit();
    BKLogSegmentWriter perStreamWriter = writer.getCachedLogWriter();
    DistributedLock lock = perStreamWriter.getLock();
    Utils.ioResult(lock.asyncClose());
    // Get second writer, steal lock
    BKAsyncLogWriter writer2 = (BKAsyncLogWriter) (dlm1.startAsyncLogSegmentNonPartitioned());
    try {
        // Succeeds, kicks off scheduked flush
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        // Succeeds, kicks off scheduled flush
        Thread.sleep(100);
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
        fail("should have thrown");
    } catch (LockingException ex) {
        LOG.debug("caught exception ", ex);
    }
    writer.close();
    dlm.close();
}

Example 29 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestAsyncReaderWriter method testSimpleAsyncReadWriteStartEmptyFactory.

/**
 * Test Case: starting reading when the streams don't exist.
 * {@link https://issues.apache.org/jira/browse/DL-42}
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 120000)
public void testSimpleAsyncReadWriteStartEmptyFactory() throws Exception {
    // int count = 50;
    int count = 1;
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(10);
    confLocal.setOutputBufferSize(1024);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 1;
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    final DistributedLogManager[] dlms = new DistributedLogManager[count];
    final TestReader[] readers = new TestReader[count];
    final CountDownLatch readyLatch = new CountDownLatch(count);
    final CountDownLatch[] syncLatches = new CountDownLatch[count];
    final CountDownLatch[] readerDoneLatches = new CountDownLatch[count];
    for (int s = 0; s < count; s++) {
        dlms[s] = namespace.openLog(name + String.format("%d", s));
        readerDoneLatches[s] = new CountDownLatch(1);
        syncLatches[s] = new CountDownLatch(numLogSegments * numRecordsPerLogSegment);
        readers[s] = new TestReader("reader-" + s, dlms[s], DLSN.InitialDLSN, false, 0, readyLatch, syncLatches[s], readerDoneLatches[s]);
        readers[s].start();
    }
    // wait all readers were positioned at least once
    readyLatch.await();
    final CountDownLatch writeLatch = new CountDownLatch(3 * count);
    final AtomicBoolean writeErrors = new AtomicBoolean(false);
    int txid = 1;
    for (long i = 0; i < 3; i++) {
        final long currentLogSegmentSeqNo = i + 1;
        BKAsyncLogWriter[] writers = new BKAsyncLogWriter[count];
        for (int s = 0; s < count; s++) {
            writers[s] = (BKAsyncLogWriter) (dlms[s].startAsyncLogSegmentNonPartitioned());
        }
        for (long j = 0; j < 1; j++) {
            final long currentEntryId = j;
            final LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid++);
            for (int s = 0; s < count; s++) {
                CompletableFuture<DLSN> dlsnFuture = writers[s].write(record);
                dlsnFuture.whenComplete(new WriteFutureEventListener(record, currentLogSegmentSeqNo, currentEntryId, writeLatch, writeErrors, true));
            }
        }
        for (int s = 0; s < count; s++) {
            writers[s].closeAndComplete();
        }
    }
    writeLatch.await();
    assertFalse("All writes should succeed", writeErrors.get());
    for (int s = 0; s < count; s++) {
        readerDoneLatches[s].await();
        assertFalse("Reader " + s + " should not encounter errors", readers[s].areErrorsFound());
        syncLatches[s].await();
        assertEquals(numLogSegments * numRecordsPerLogSegment, readers[s].getNumReads().get());
        assertTrue("Reader " + s + " should position at least once", readers[s].getNumReaderPositions().get() > 0);
    }
    for (int s = 0; s < count; s++) {
        readers[s].stop();
        dlms[s].close();
    }
}

Example 30 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method testNamespaceListener.

@Test(timeout = 60000)
public void testNamespaceListener() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    final CountDownLatch[] latches = new CountDownLatch[3];
    for (int i = 0; i < 3; i++) {
        latches[i] = new CountDownLatch(1);
    }
    final AtomicInteger numUpdates = new AtomicInteger(0);
    final AtomicInteger numFailures = new AtomicInteger(0);
    final AtomicReference<Collection<String>> receivedStreams = new AtomicReference<Collection<String>>(null);
    namespace.registerNamespaceListener(new NamespaceListener() {

        @Override
        public void onStreamsChanged(Iterator<String> streams) {
            Set<String> streamSet = Sets.newHashSet(streams);
            int updates = numUpdates.incrementAndGet();
            if (streamSet.size() != updates - 1) {
                numFailures.incrementAndGet();
            }
            receivedStreams.set(streamSet);
            latches[updates - 1].countDown();
        }
    });
    latches[0].await();
    namespace.createLog("test1");
    latches[1].await();
    namespace.createLog("test2");
    latches[2].await();
    assertEquals(0, numFailures.get());
    assertNotNull(receivedStreams.get());
    Set<String> streamSet = new HashSet<String>();
    streamSet.addAll(receivedStreams.get());
    assertEquals(2, receivedStreams.get().size());
    assertEquals(2, streamSet.size());
    assertTrue(streamSet.contains("test1"));
    assertTrue(streamSet.contains("test2"));
}