Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 71 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
 * {@link https://issues.apache.org/jira/browse/DL-44}.
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogConfiguration readConf = new DistributedLogConfiguration();
    readConf.addConfiguration(conf);
    readConf.setLogSegmentCacheEnabled(false);
    readConf.setLogSegmentSequenceNumberValidationEnabled(true);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    Namespace readNamespace = NamespaceBuilder.newBuilder().conf(readConf).uri(uri).build();
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = readNamespace.openLog(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    DLSN lastDLSN = DLSN.InitialDLSN;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
        lastDLSN = record.getDlsn();
    }
    LOG.info("Injecting bad log segment '3'");
    dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    LOG.info("Injected bad log segment '3'");
    // there isn't records should be read
    CompletableFuture<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        LogRecordWithDLSN record = Utils.ioResult(readFuture);
        fail("Should fail reading next record " + record + " when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Dryrun fix inprogress segment that has lower sequence number");
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    try {
        reader = readDLM.getAsyncLogReader(lastDLSN);
        Utils.ioResult(reader.readNext());
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Actual run fix inprogress segment that has lower sequence number");
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    // be able to read more after fix
    reader = readDLM.getAsyncLogReader(lastDLSN);
    // skip the first record
    Utils.ioResult(reader.readNext());
    readFuture = reader.readNext();
    expectedTxId = 51L;
    LogRecord record = Utils.ioResult(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    namespace.close();
    readNamespace.close();
}

Example 72 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestDistributedLogTool method setupDefaults.

@BeforeClass
public static void setupDefaults() throws Exception {
    defaultUri = DLMTestUtil.createDLMURI(zkPort, defaultPath);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("DefaultStream", conf, defaultUri);
    bindStream(defaultUri, defaultLedgerPath, defaultHost);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 8192);
    dlm.close();
}

Example 73 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestDistributedLogTool method testToolTruncate.

@Test(timeout = 60000)
public void testToolTruncate() throws Exception {
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("TruncateStream", conf, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 1000);
    dlm.close();
    TruncateCommand cmd = new TruncateCommand();
    cmd.setUri(defaultUri);
    cmd.setFilter("TruncateStream");
    cmd.setForce(true);
    assertEquals(0, cmd.runCmd());
}

Example 74 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class DLFileSystem method open.

@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
    try {
        DistributedLogManager dlm = namespace.openLog(getStreamName(path));
        LogReader reader;
        try {
            reader = dlm.openLogReader(DLSN.InitialDLSN);
        } catch (LogNotFoundException lnfe) {
            throw new FileNotFoundException(path.toString());
        } catch (LogEmptyException lee) {
            throw new FileNotFoundException(path.toString());
        }
        return new FSDataInputStream(new BufferedFSInputStream(new DLInputStream(dlm, reader, 0L), bufferSize));
    } catch (LogNotFoundException e) {
        throw new FileNotFoundException(path.toString());
    }
}

Example 75 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project incubator-pulsar by apache.

the class DLInputStreamTest method testReadEos.

/**
 * Test Case: reader hits eos (end of stream)
 */
@Test
public void testReadEos() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);
    when(reader.readNext(anyBoolean())).thenThrow(new EndOfStreamException("eos"));
    byte[] b = new byte[1];
    DLInputStream in = new DLInputStream(dlm);
    assertEquals("Should return 0 when reading an empty eos stream", 0, in.read(b, 0, 1));
    assertEquals("Should return -1 when reading an empty eos stream", -1, in.read(b, 0, 1));
}