Examples with DynamicDistributedLogConfiguration com.twitter.distributedlog.config.DynamicDistributedLogConfiguration used on opensource projects

Example 1 with DynamicDistributedLogConfiguration

use of com.twitter.distributedlog.config.DynamicDistributedLogConfiguration in project distributedlog by twitter.

the class TestAsyncReaderWriter method testCreateLogStreamWithDifferentReplicationFactor.

@Test(timeout = 60000)
public void testCreateLogStreamWithDifferentReplicationFactor() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    ConcurrentBaseConfiguration baseConf = new ConcurrentConstConfiguration(confLocal);
    DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(baseConf);
    dynConf.setProperty(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE, DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    // use the pool
    DistributedLogManager dlm = namespace.openLog(name + "-pool");
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    long ledgerId = segments.get(0).getLedgerId();
    LedgerHandle lh = ((BKDistributedLogNamespace) namespace).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    LedgerMetadata metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    // use customized configuration
    dlm = namespace.openLog(name + "-custom", Optional.<DistributedLogConfiguration>absent(), Optional.of(dynConf));
    writer = dlm.startAsyncLogSegmentNonPartitioned();
    FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    ledgerId = segments.get(0).getLedgerId();
    lh = ((BKDistributedLogNamespace) namespace).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    namespace.close();
}

Example 2 with DynamicDistributedLogConfiguration

use of com.twitter.distributedlog.config.DynamicDistributedLogConfiguration in project distributedlog by twitter.

the class DistributedLogServer method runServer.

public void runServer() throws ConfigurationException, IllegalArgumentException, IOException {
    if (!uri.isPresent()) {
        throw new IllegalArgumentException("No distributedlog uri provided.");
    }
    URI dlUri = URI.create(uri.get());
    DistributedLogConfiguration dlConf = new DistributedLogConfiguration();
    if (conf.isPresent()) {
        String configFile = conf.get();
        try {
            dlConf.loadConf(new File(configFile).toURI().toURL());
        } catch (ConfigurationException e) {
            throw new IllegalArgumentException("Failed to load distributedlog configuration from " + configFile + ".");
        } catch (MalformedURLException e) {
            throw new IllegalArgumentException("Failed to load distributedlog configuration from malformed " + configFile + ".");
        }
    }
    this.configExecutorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("DistributedLogService-Dyncfg-%d").setDaemon(true).build());
    // server configuration and dynamic configuration
    ServerConfiguration serverConf = new ServerConfiguration();
    serverConf.loadConf(dlConf);
    // overwrite the shard id if it is provided in the args
    if (shardId.isPresent()) {
        serverConf.setServerShardId(shardId.get());
    }
    serverConf.validate();
    DynamicDistributedLogConfiguration dynDlConf = getServiceDynConf(dlConf);
    logger.info("Starting stats provider : {}", statsProvider.getClass());
    statsProvider.start(dlConf);
    if (announceServerSet.isPresent() && announceServerSet.get()) {
        announcer = new ServerSetAnnouncer(dlUri, port.or(0), statsPort.or(0), shardId.or(0));
    } else {
        announcer = new NOPAnnouncer();
    }
    // Build the stream partition converter
    StreamPartitionConverter converter;
    try {
        converter = ReflectionUtils.newInstance(serverConf.getStreamPartitionConverterClass());
    } catch (ConfigurationException e) {
        logger.warn("Failed to load configured stream-to-partition converter. Fallback to use {}", IdentityStreamPartitionConverter.class.getName());
        converter = new IdentityStreamPartitionConverter();
    }
    StreamConfigProvider streamConfProvider = getStreamConfigProvider(dlConf, converter);
    // pre-run
    preRun(dlConf, serverConf);
    Pair<DistributedLogServiceImpl, Server> serverPair = runServer(serverConf, dlConf, dynDlConf, dlUri, converter, statsProvider, port.or(0), keepAliveLatch, statsReceiver, thriftmux.isPresent(), streamConfProvider);
    this.dlService = serverPair.getLeft();
    this.server = serverPair.getRight();
    // announce the service
    announcer.announce();
}

Example 3 with DynamicDistributedLogConfiguration

use of com.twitter.distributedlog.config.DynamicDistributedLogConfiguration in project distributedlog by twitter.

the class BKDistributedLogNamespace method createDistributedLogManager.

/**
     * Open the log in location <i>uri</i>.
     *
     * @param uri
     *          location to store the log
     * @param nameOfLogStream
     *          name of the log
     * @param clientSharingOption
     *          client sharing option
     * @param logConfiguration
     *          optional stream configuration
     * @param dynamicLogConfiguration
     *          dynamic stream configuration overrides.
     * @return distributedlog manager instance.
     * @throws InvalidStreamNameException if the stream name is invalid
     * @throws IOException
     */
protected DistributedLogManager createDistributedLogManager(URI uri, String nameOfLogStream, ClientSharingOption clientSharingOption, Optional<DistributedLogConfiguration> logConfiguration, Optional<DynamicDistributedLogConfiguration> dynamicLogConfiguration) throws InvalidStreamNameException, IOException {
    // Make sure the name is well formed
    validateName(nameOfLogStream);
    DistributedLogConfiguration mergedConfiguration = new DistributedLogConfiguration();
    mergedConfiguration.addConfiguration(conf);
    mergedConfiguration.loadStreamConf(logConfiguration);
    // If dynamic config was not provided, default to a static view of the global configuration.
    DynamicDistributedLogConfiguration dynConf = null;
    if (dynamicLogConfiguration.isPresent()) {
        dynConf = dynamicLogConfiguration.get();
    } else {
        dynConf = ConfUtils.getConstDynConf(mergedConfiguration);
    }
    ZooKeeperClientBuilder writerZKCBuilderForDL = null;
    ZooKeeperClientBuilder readerZKCBuilderForDL = null;
    ZooKeeperClient writerZKCForBK = null;
    ZooKeeperClient readerZKCForBK = null;
    BookKeeperClientBuilder writerBKCBuilder = null;
    BookKeeperClientBuilder readerBKCBuilder = null;
    switch(clientSharingOption) {
        case SharedClients:
            writerZKCBuilderForDL = sharedWriterZKCBuilderForDL;
            readerZKCBuilderForDL = sharedReaderZKCBuilderForDL;
            writerBKCBuilder = sharedWriterBKCBuilder;
            readerBKCBuilder = sharedReaderBKCBuilder;
            break;
        case SharedZKClientPerStreamBKClient:
            writerZKCBuilderForDL = sharedWriterZKCBuilderForDL;
            readerZKCBuilderForDL = sharedReaderZKCBuilderForDL;
            synchronized (this) {
                if (null == this.sharedWriterZKCForBK) {
                    this.sharedWriterZKCBuilderForBK = createBKZKClientBuilder(String.format("bkzk:%s:factory_writer_shared", uri), mergedConfiguration, bkdlConfig.getBkZkServersForWriter(), statsLogger.scope("bkzk_factory_writer_shared"));
                    this.sharedWriterZKCForBK = this.sharedWriterZKCBuilderForBK.build();
                }
                if (null == this.sharedReaderZKCForBK) {
                    if (bkdlConfig.getBkZkServersForWriter().equals(bkdlConfig.getBkZkServersForReader())) {
                        this.sharedReaderZKCBuilderForBK = this.sharedWriterZKCBuilderForBK;
                    } else {
                        this.sharedReaderZKCBuilderForBK = createBKZKClientBuilder(String.format("bkzk:%s:factory_reader_shared", uri), mergedConfiguration, bkdlConfig.getBkZkServersForReader(), statsLogger.scope("bkzk_factory_reader_shared"));
                    }
                    this.sharedReaderZKCForBK = this.sharedReaderZKCBuilderForBK.build();
                }
                writerZKCForBK = this.sharedWriterZKCForBK;
                readerZKCForBK = this.sharedReaderZKCForBK;
            }
            break;
    }
    LedgerAllocator dlmLedgerAlloctor = null;
    PermitManager dlmLogSegmentRollingPermitManager = PermitManager.UNLIMITED_PERMIT_MANAGER;
    if (ClientSharingOption.SharedClients == clientSharingOption) {
        dlmLedgerAlloctor = this.allocator;
        dlmLogSegmentRollingPermitManager = this.logSegmentRollingPermitManager;
    }
    return new BKDistributedLogManager(nameOfLogStream, /* Log Name */
    mergedConfiguration, /* Configuration */
    dynConf, /* Dynamic Configuration */
    uri, /* Namespace */
    writerZKCBuilderForDL, /* ZKC Builder for DL Writer */
    readerZKCBuilderForDL, /* ZKC Builder for DL Reader */
    writerZKCForBK, /* ZKC for BookKeeper for DL Writers */
    readerZKCForBK, /* ZKC for BookKeeper for DL Readers */
    writerBKCBuilder, /* BookKeeper Builder for DL Writers */
    readerBKCBuilder, /* BookKeeper Builder for DL Readers */
    lockFactory, /* Lock Factory */
    writerSegmentMetadataStore, /* Log Segment Metadata Store for DL Writers */
    readerSegmentMetadataStore, /* Log Segment Metadata Store for DL Readers */
    scheduler, /* DL scheduler */
    readAheadExecutor, /* Read Aheader Executor */
    lockStateExecutor, /* Lock State Executor */
    channelFactory, /* Netty Channel Factory */
    requestTimer, /* Request Timer */
    readAheadExceptionsLogger, /* ReadAhead Exceptions Logger */
    clientId, /* Client Id */
    regionId, /* Region Id */
    dlmLedgerAlloctor, /* Ledger Allocator */
    writeLimiter, /* Write Limiter */
    dlmLogSegmentRollingPermitManager, /* Log segment rolling limiter */
    featureProvider.scope("dl"), /* Feature Provider */
    statsLogger, /* Stats Logger */
    perLogStatsLogger);
}

Example 4 with DynamicDistributedLogConfiguration

use of com.twitter.distributedlog.config.DynamicDistributedLogConfiguration in project distributedlog by twitter.

the class StreamManagerImpl method getOrCreateStream.

@Override
public Stream getOrCreateStream(String streamName) throws IOException {
    Stream stream = streams.get(streamName);
    if (null == stream) {
        closeLock.readLock().lock();
        try {
            if (closed) {
                return null;
            }
            DynamicDistributedLogConfiguration dynConf = getDynConf(streamName);
            int maxCachedPartitions = dynConf.getMaxCachedPartitionsPerProxy();
            // get partition from the stream name
            Partition partition = partitionConverter.convert(streamName);
            // add partition to cached map
            if (!cachedPartitions.addPartition(partition, maxCachedPartitions)) {
                throw new StreamUnavailableException("Stream " + streamName + " is not allowed to cache more than " + maxCachedPartitions + " partitions");
            }
            stream = newStream(streamName, dynConf);
            Stream oldWriter = streams.putIfAbsent(streamName, stream);
            if (null != oldWriter) {
                stream = oldWriter;
            } else {
                numCached.getAndIncrement();
                logger.info("Inserted mapping stream name {} -> stream {}", streamName, stream);
                stream.initialize();
                stream.start();
            }
        } finally {
            closeLock.readLock().unlock();
        }
    }
    return stream;
}

Example 5 with DynamicDistributedLogConfiguration

use of com.twitter.distributedlog.config.DynamicDistributedLogConfiguration in project distributedlog by twitter.

the class TestStreamManager method testCreateStream.

@Test
public void testCreateStream() throws Exception {
    Stream mockStream = mock(Stream.class);
    final String streamName = "stream1";
    when(mockStream.getStreamName()).thenReturn(streamName);
    StreamFactory mockStreamFactory = mock(StreamFactory.class);
    StreamPartitionConverter mockPartitionConverter = mock(StreamPartitionConverter.class);
    StreamConfigProvider mockStreamConfigProvider = mock(StreamConfigProvider.class);
    when(mockStreamFactory.create((String) any(), (DynamicDistributedLogConfiguration) any(), (StreamManager) any())).thenReturn(mockStream);
    DistributedLogNamespace dlNamespace = mock(DistributedLogNamespace.class);
    ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1);
    StreamManager streamManager = new StreamManagerImpl("", new DistributedLogConfiguration(), executorService, mockStreamFactory, mockPartitionConverter, mockStreamConfigProvider, dlNamespace);
    assertTrue(Await.ready(streamManager.createStreamAsync(streamName)).isReturn());
    verify(dlNamespace).createLog(streamName);
}