Examples with LogReader org.apache.distributedlog.api.LogReader used on opensource projects

Example 26 with LogReader

use of org.apache.distributedlog.api.LogReader in project incubator-heron by apache.

the class DLDownloaderTest method testDownload.

@Test
public void testDownload() throws Exception {
    String logName = "test-download";
    URI uri = URI.create("distributedlog://127.0.0.1/test/distributedlog/" + logName);
    File tempFile = File.createTempFile("test", "download");
    // make sure it is deleted when the test completes
    tempFile.deleteOnExit();
    Path path = Paths.get(tempFile.toURI());
    Namespace ns = mock(Namespace.class);
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(ns.openLog(anyString())).thenReturn(dlm);
    when(dlm.getInputStream(eq(DLSN.InitialDLSN))).thenReturn(reader);
    when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));
    NamespaceBuilder nsBuilder = mock(NamespaceBuilder.class);
    when(nsBuilder.clientId(anyString())).thenReturn(nsBuilder);
    when(nsBuilder.conf(any(DistributedLogConfiguration.class))).thenReturn(nsBuilder);
    when(nsBuilder.uri(any(URI.class))).thenReturn(nsBuilder);
    when(nsBuilder.build()).thenReturn(ns);
    PowerMockito.mockStatic(Extractor.class);
    PowerMockito.doNothing().when(Extractor.class, "extract", any(InputStream.class), any(Path.class));
    DLDownloader downloader = new DLDownloader(() -> nsBuilder);
    downloader.download(uri, path);
    URI parentUri = URI.create("distributedlog://127.0.0.1/test/distributedlog");
    verify(nsBuilder, times(1)).clientId(eq("heron-downloader"));
    verify(nsBuilder, times(1)).conf(eq(CONF));
    verify(nsBuilder, times(1)).uri(parentUri);
    PowerMockito.verifyStatic(times(1));
    Extractor.extract(any(InputStream.class), eq(path));
    verify(ns, times(1)).openLog(eq(logName));
    verify(ns, times(1)).close();
}

Example 27 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKSyncLogReader method testCreateReaderBeyondLastTransactionId.

@Test(timeout = 60000)
public void testCreateReaderBeyondLastTransactionId() throws Exception {
    String name = testName.getMethodName();
    DistributedLogManager dlm = createNewDLM(conf, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1; i < 10; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();
    LogReader reader = dlm.getInputStream(20L);
    assertNull(reader.readNext(false));
    // write another 20 records
    out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 10; i < 30; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();
    for (int i = 0; i < 10; i++) {
        LogRecord record = waitForNextRecord(reader);
        assertEquals(20L + i, record.getTransactionId());
    }
    assertNull(reader.readNext(false));
}

Example 28 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestBKSyncLogReader method testReadingFromEmptyLog.

@Test(timeout = 60000)
public void testReadingFromEmptyLog() throws Exception {
    String name = testName.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    // write a record but not commit
    LogRecord op = DLMTestUtil.getLogRecordInstance(1L);
    out.write(op);
    LogReader reader = dlm.getInputStream(1L);
    assertNull(reader.readNext(true));
    assertNull(reader.readNext(false));
    op = DLMTestUtil.getLogRecordInstance(2L);
    out.write(op);
    // reader is able to read first record
    LogRecord record = waitForNextRecord(reader);
    assertNotNull(record);
    assertEquals(1L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);
    assertNull(reader.readNext(true));
    out.close();
    reader.close();
    dlm.close();
}

Example 29 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestInterleavedReaders method testInterleavedReaders.

@Test(timeout = 60000)
public void testInterleavedReaders() throws Exception {
    String name = "distrlog-interleaved";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    BKDistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = null;
    LogReader reader1 = null;
    long txid = 1;
    int numTrans = 0;
    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 4; j++) {
        for (int k = 1; k <= 10; k++) {
            Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        if (null == reader0) {
            reader0 = dlmreader0.getInputStream(1);
        }
        if (null == reader1) {
            reader1 = dlmreader1.getInputStream(1);
        }
        numTrans += drainStreams(reader0, 10, reader1, 10);
        assertEquals((txid - 1), numTrans);
    }
    reader0.close();
    reader1.close();
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}

Example 30 with LogReader

use of org.apache.distributedlog.api.LogReader in project bookkeeper by apache.

the class TestInterleavedReaders method testInterleavedReadersWithRecovery.

@Test(timeout = 60000)
public void testInterleavedReadersWithRecovery() throws Exception {
    String name = "distrlog-interleaved-recovery";
    BKDistributedLogManager dlmwrite0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmreader0 = createNewDLM(conf, name + "-0");
    BKDistributedLogManager dlmwrite1 = createNewDLM(conf, name + "-1");
    BKDistributedLogManager dlmreader1 = createNewDLM(conf, name + "-1");
    LogReader reader0 = null;
    LogReader reader1 = null;
    long txid = 1;
    int numTrans = 0;
    BKAsyncLogWriter writer0 = dlmwrite0.startAsyncLogSegmentNonPartitioned();
    BKAsyncLogWriter writer1 = dlmwrite1.startAsyncLogSegmentNonPartitioned();
    for (long j = 1; j <= 2; j++) {
        for (int k = 1; k <= 6; k++) {
            if (k == 3) {
                writer0.setForceRecovery(true);
                writer1.setForceRecovery(true);
            }
            DLSN dlsn1 = Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer1 write record {} - txid = {}", dlsn1, txid - 1);
            DLSN dlsn0 = Utils.ioResult(writer0.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LOG.info("writer0 write record {} - txid = {}", dlsn0, txid - 1);
            writer0.setForceRecovery(false);
            writer1.setForceRecovery(false);
        }
        Utils.ioResult(writer1.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        Utils.ioResult(writer0.writeControlRecord(DLMTestUtil.getLogRecordInstance(txid - 1)));
        if (null == reader0) {
            reader0 = dlmreader0.getInputStream(1);
        }
        if (null == reader1) {
            reader1 = dlmreader1.getInputStream(1);
        }
        numTrans += drainStreams(reader0, 6, reader1, 6);
        assertEquals((txid - 1), numTrans);
    }
    reader0.close();
    reader1.close();
    assertEquals(txid - 1, dlmreader0.getLogRecordCount() + dlmreader1.getLogRecordCount());
    dlmreader0.close();
    dlmwrite0.close();
    dlmreader1.close();
    dlmwrite1.close();
}