Examples with DistributedLogConfiguration org.apache.distributedlog.DistributedLogConfiguration used on opensource projects

Example 36 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestDistributedLogTool method testToolTruncateStream.

@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", confLocal, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 3, 1000);
    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);
    assertEquals(0, cmd.runCmd());
    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());
    reader.close();
    dlm.close();
}

Example 37 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class DLFileSystem method create.

@Override
public FSDataOutputStream create(Path path, FsPermission fsPermission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progressable) throws IOException {
    // for overwrite, delete the existing file first.
    if (overwrite) {
        delete(path, false);
    }
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(dlConf);
    confLocal.setEnsembleSize(replication);
    confLocal.setWriteQuorumSize(replication);
    confLocal.setAckQuorumSize(replication);
    confLocal.setMaxLogSegmentBytes(blockSize);
    return append(path, bufferSize, Optional.of(confLocal));
}

Example 38 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project incubator-heron by apache.

the class DLUploader method initializeNamespace.

private void initializeNamespace(Config upConfig) throws IOException {
    int numReplicas = DLContext.dlTopologiesNumReplicas(upConfig);
    DistributedLogConfiguration conf = new DistributedLogConfiguration().setWriteLockEnabled(false).setOutputBufferSize(// 256k
    256 * 1024).setPeriodicFlushFrequencyMilliSeconds(// disable periodical flush
    0).setImmediateFlushEnabled(// disable immediate flush
    false).setLogSegmentRollingIntervalMinutes(// disable time-based rolling
    0).setMaxLogSegmentBytes(// disable size-based rolling
    Long.MAX_VALUE).setExplicitTruncationByApplication(// no auto-truncation
    true).setRetentionPeriodHours(// long retention
    Integer.MAX_VALUE).setEnsembleSize(// replica settings
    numReplicas).setWriteQuorumSize(numReplicas).setAckQuorumSize(numReplicas).setUseDaemonThread(// use daemon thread
    true);
    URI uri = URI.create(DLContext.dlTopologiesNamespaceURI(this.config));
    LOG.info(String.format("Initializing distributedlog namespace for uploading topologies : %s", uri));
    this.namespace = this.nsBuilderSupplier.get().clientId("heron-uploader").conf(conf).uri(uri).build();
}

Example 39 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project incubator-pulsar by apache.

the class WorkerService method start.

private void start(URI dlogUri) throws InterruptedException {
    log.info("Starting worker {}...", workerConfig.getWorkerId());
    try {
        log.info("Worker Configs: {}", new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(workerConfig));
    } catch (JsonProcessingException e) {
        log.warn("Failed to print worker configs with error {}", e.getMessage(), e);
    }
    // create the dlog namespace for storing function packages
    DistributedLogConfiguration dlogConf = Utils.getDlogConf(workerConfig);
    try {
        this.dlogNamespace = NamespaceBuilder.newBuilder().conf(dlogConf).clientId("function-worker-" + workerConfig.getWorkerId()).uri(dlogUri).build();
    } catch (Exception e) {
        log.error("Failed to initialize dlog namespace {} for storing function packages", dlogUri, e);
        throw new RuntimeException(e);
    }
    // initialize the function metadata manager
    try {
        this.client = PulsarClient.create(this.workerConfig.getPulsarServiceUrl());
        log.info("Created Pulsar client");
        // create scheduler manager
        this.schedulerManager = new SchedulerManager(this.workerConfig, this.client);
        // create function meta data manager
        this.functionMetaDataManager = new FunctionMetaDataManager(this.workerConfig, this.schedulerManager, this.client);
        // create membership manager
        this.membershipManager = new MembershipManager(this.workerConfig, this.client);
        // create function runtime manager
        this.functionRuntimeManager = new FunctionRuntimeManager(this.workerConfig, this.client, this.dlogNamespace, this.membershipManager);
        // Setting references to managers in scheduler
        this.schedulerManager.setFunctionMetaDataManager(this.functionMetaDataManager);
        this.schedulerManager.setFunctionRuntimeManager(this.functionRuntimeManager);
        this.schedulerManager.setMembershipManager(this.membershipManager);
        // initialize function metadata manager
        this.functionMetaDataManager.initialize();
        // Starting cluster services
        log.info("Start cluster services...");
        this.clusterServiceCoordinator = new ClusterServiceCoordinator(this.workerConfig.getWorkerId(), membershipManager);
        this.clusterServiceCoordinator.addTask("membership-monitor", this.workerConfig.getFailureCheckFreqMs(), () -> membershipManager.checkFailures(functionMetaDataManager, functionRuntimeManager, schedulerManager));
        this.clusterServiceCoordinator.start();
        // Start function runtime manager
        this.functionRuntimeManager.start();
    } catch (Exception e) {
        log.error("Error Starting up in worker", e);
        throw new RuntimeException(e);
    }
}