Examples with DistributedLogConfiguration org.apache.distributedlog.DistributedLogConfiguration used on opensource projects

Example 16 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestDistributedLogAdmin method testChangeSequenceNumber.

/**
 * {@link https://issues.apache.org/jira/browse/DL-44}.
 */
@DistributedLogAnnotations.FlakyTest
@Ignore
@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testChangeSequenceNumber() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogConfiguration readConf = new DistributedLogConfiguration();
    readConf.addConfiguration(conf);
    readConf.setLogSegmentCacheEnabled(false);
    readConf.setLogSegmentSequenceNumberValidationEnabled(true);
    URI uri = createDLMURI("/change-sequence-number");
    zooKeeperClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    Namespace readNamespace = NamespaceBuilder.newBuilder().conf(readConf).uri(uri).build();
    String streamName = "change-sequence-number";
    // create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 4, 10);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 5, 41, false, 10, true);
    dlm.close();
    // create a reader
    DistributedLogManager readDLM = readNamespace.openLog(streamName);
    AsyncLogReader reader = readDLM.getAsyncLogReader(DLSN.InitialDLSN);
    // read the records
    long expectedTxId = 1L;
    DLSN lastDLSN = DLSN.InitialDLSN;
    for (int i = 0; i < 4 * 10; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
        lastDLSN = record.getDlsn();
    }
    LOG.info("Injecting bad log segment '3'");
    dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithGivenLogSegmentSeqNo(dlm, confLocal, 3L, 5 * 10 + 1, true, 10, false);
    LOG.info("Injected bad log segment '3'");
    // there isn't records should be read
    CompletableFuture<LogRecordWithDLSN> readFuture = reader.readNext();
    try {
        LogRecordWithDLSN record = Utils.ioResult(readFuture);
        fail("Should fail reading next record " + record + " when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Dryrun fix inprogress segment that has lower sequence number");
    // Dryrun
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    try {
        reader = readDLM.getAsyncLogReader(lastDLSN);
        Utils.ioResult(reader.readNext());
        fail("Should fail reading next when there is a corrupted log segment");
    } catch (UnexpectedException ue) {
    // expected
    }
    LOG.info("Actual run fix inprogress segment that has lower sequence number");
    // Actual run
    DistributedLogAdmin.fixInprogressSegmentWithLowerSequenceNumber(namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), streamName, false, false);
    // be able to read more after fix
    reader = readDLM.getAsyncLogReader(lastDLSN);
    // skip the first record
    Utils.ioResult(reader.readNext());
    readFuture = reader.readNext();
    expectedTxId = 51L;
    LogRecord record = Utils.ioResult(readFuture);
    assertNotNull(record);
    DLMTestUtil.verifyLogRecord(record);
    assertEquals(expectedTxId, record.getTransactionId());
    expectedTxId++;
    for (int i = 1; i < 10; i++) {
        record = Utils.ioResult(reader.readNext());
        assertNotNull(record);
        DLMTestUtil.verifyLogRecord(record);
        assertEquals(expectedTxId, record.getTransactionId());
        expectedTxId++;
    }
    Utils.close(reader);
    readDLM.close();
    dlm.close();
    namespace.close();
    readNamespace.close();
}

Example 17 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestDynamicConfigurationFactory method getConfigFactory.

private DynamicConfigurationFactory getConfigFactory(File configFile) {
    String streamConfigPath = configFile.getParent();
    ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1);
    ConcurrentBaseConfiguration defaultConf = new ConcurrentConstConfiguration(new DistributedLogConfiguration());
    return new DynamicConfigurationFactory(executorService, 100, TimeUnit.MILLISECONDS);
}

Example 18 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestZKLogMetadataStore method setup.

@Before
public void setup() throws Exception {
    zkc = TestZooKeeperClientBuilder.newBuilder().uri(createDLMURI("/")).sessionTimeoutMs(zkSessionTimeoutMs).build();
    scheduler = OrderedScheduler.newSchedulerBuilder().name("test-zk-logmetadata-store").numThreads(1).build();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.addConfiguration(baseConf);
    this.uri = createDLMURI("/" + runtime.getMethodName());
    metadataStore = new ZKLogMetadataStore(conf, uri, zkc, scheduler);
}

Example 19 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class StreamCluster method startServer.

private LifecycleComponent startServer() throws Exception {
    int bookiePort;
    int grpcPort;
    boolean success = false;
    int retries = 0;
    while (!success) {
        synchronized (this) {
            bookiePort = nextBookiePort++;
            grpcPort = nextGrpcPort++;
        }
        LifecycleComponent server = null;
        try {
            ServerConfiguration serverConf = new ServerConfiguration();
            serverConf.loadConf(baseConf);
            serverConf.setBookiePort(bookiePort);
            File bkDir = new File(spec.storageRootDir(), "bookie_" + bookiePort);
            serverConf.setJournalDirName(bkDir.getPath());
            serverConf.setLedgerDirNames(new String[] { bkDir.getPath() });
            DistributedLogConfiguration dlConf = new DistributedLogConfiguration();
            dlConf.loadConf(serverConf);
            File rangesStoreDir = new File(spec.storageRootDir(), "ranges_" + grpcPort);
            StorageConfiguration storageConf = new StorageConfiguration(serverConf);
            storageConf.setRangeStoreDirNames(new String[] { rangesStoreDir.getPath() });
            storageConf.setServeReadOnlyTables(spec.serveReadOnlyTable);
            log.info("Attempting to start storage server at (bookie port = {}, grpc port = {})" + " : bkDir = {}, rangesStoreDir = {}, serveReadOnlyTables = {}", bookiePort, grpcPort, bkDir, rangesStoreDir, spec.serveReadOnlyTable);
            server = StorageServer.startStorageServer(serverConf, grpcPort, spec.numServers() * 2, Optional.empty());
            server.start();
            log.info("Started storage server at (bookie port = {}, grpc port = {})", bookiePort, grpcPort);
            this.rpcEndpoints.add(StorageServer.createLocalEndpoint(grpcPort, false));
            return server;
        } catch (Throwable e) {
            log.error("Failed to start storage server", e);
            if (null != server) {
                server.stop();
            }
            if (e.getCause() instanceof BindException) {
                retries++;
                if (retries > MAX_RETRIES) {
                    throw (BindException) e.getCause();
                }
            } else {
                throw e;
            }
        }
    }
    throw new IOException("Failed to start any storage server.");
}

Example 20 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project incubator-pulsar by apache.

the class Utils method getDlogConf.

public static DistributedLogConfiguration getDlogConf(WorkerConfig workerConfig) {
    int numReplicas = workerConfig.getNumFunctionPackageReplicas();
    DistributedLogConfiguration conf = new DistributedLogConfiguration().setWriteLockEnabled(false).setOutputBufferSize(// 256k
    256 * 1024).setPeriodicFlushFrequencyMilliSeconds(// disable periodical flush
    0).setImmediateFlushEnabled(// disable immediate flush
    false).setLogSegmentRollingIntervalMinutes(// disable time-based rolling
    0).setMaxLogSegmentBytes(// disable size-based rolling
    Long.MAX_VALUE).setExplicitTruncationByApplication(// no auto-truncation
    true).setRetentionPeriodHours(// long retention
    Integer.MAX_VALUE).setEnsembleSize(// replica settings
    numReplicas).setWriteQuorumSize(numReplicas).setAckQuorumSize(numReplicas).setUseDaemonThread(true);
    conf.setProperty("bkc.allowShadedLedgerManagerFactoryClass", true);
    conf.setProperty("bkc.shadedLedgerManagerFactoryClassPrefix", "dlshade.");
    return conf;
}