Examples with LedgerAllocator com.twitter.distributedlog.bk.LedgerAllocator used on opensource projects

Example 1 with LedgerAllocator

use of com.twitter.distributedlog.bk.LedgerAllocator in project distributedlog by twitter.

the class BKDistributedLogManager method createLedgerAllocator.

// Create Ledger Allocator
LedgerAllocator createLedgerAllocator(ZKLogMetadataForWriter logMetadata) throws IOException {
    LedgerAllocator ledgerAllocatorDelegator;
    if (!dynConf.getEnableLedgerAllocatorPool()) {
        QuorumConfigProvider quorumConfigProvider = new DynamicQuorumConfigProvider(dynConf);
        LedgerAllocator allocator = new SimpleLedgerAllocator(logMetadata.getAllocationPath(), logMetadata.getAllocationData(), quorumConfigProvider, writerZKC, writerBKC);
        ledgerAllocatorDelegator = new LedgerAllocatorDelegator(allocator, true);
    } else {
        ledgerAllocatorDelegator = ledgerAllocator;
    }
    return ledgerAllocatorDelegator;
}

Example 2 with LedgerAllocator

use of com.twitter.distributedlog.bk.LedgerAllocator in project distributedlog by twitter.

the class BKDistributedLogManager method createWriteHandler.

private void createWriteHandler(ZKLogMetadataForWriter logMetadata, boolean lockHandler, final Promise<BKLogWriteHandler> createPromise) {
    OrderedScheduler lockStateExecutor = getLockStateExecutor(true);
    // Build the locks
    DistributedLock lock;
    if (conf.isWriteLockEnabled()) {
        lock = new ZKDistributedLock(lockStateExecutor, getLockFactory(true), logMetadata.getLockPath(), conf.getLockTimeoutMilliSeconds(), statsLogger);
    } else {
        lock = NopDistributedLock.INSTANCE;
    }
    // Build the ledger allocator
    LedgerAllocator allocator;
    try {
        allocator = createLedgerAllocator(logMetadata);
    } catch (IOException e) {
        FutureUtils.setException(createPromise, e);
        return;
    }
    // Make sure writer handler created before resources are initialized
    final BKLogWriteHandler writeHandler = new BKLogWriteHandler(logMetadata, conf, writerZKCBuilder, writerBKCBuilder, writerMetadataStore, scheduler, allocator, statsLogger, perLogStatsLogger, alertStatsLogger, clientId, regionId, writeLimiter, featureProvider, dynConf, lock);
    PermitManager manager = getLogSegmentRollingPermitManager();
    if (manager instanceof Watcher) {
        writeHandler.register((Watcher) manager);
    }
    if (lockHandler) {
        writeHandler.lockHandler().addEventListener(new FutureEventListener<DistributedLock>() {

            @Override
            public void onSuccess(DistributedLock lock) {
                FutureUtils.setValue(createPromise, writeHandler);
            }

            @Override
            public void onFailure(final Throwable cause) {
                writeHandler.asyncClose().ensure(new AbstractFunction0<BoxedUnit>() {

                    @Override
                    public BoxedUnit apply() {
                        FutureUtils.setException(createPromise, cause);
                        return BoxedUnit.UNIT;
                    }
                });
            }
        });
    } else {
        FutureUtils.setValue(createPromise, writeHandler);
    }
}

Example 3 with LedgerAllocator

use of com.twitter.distributedlog.bk.LedgerAllocator in project distributedlog by twitter.

the class BKDistributedLogNamespace method createDistributedLogManager.

/**
     * Open the log in location <i>uri</i>.
     *
     * @param uri
     *          location to store the log
     * @param nameOfLogStream
     *          name of the log
     * @param clientSharingOption
     *          client sharing option
     * @param logConfiguration
     *          optional stream configuration
     * @param dynamicLogConfiguration
     *          dynamic stream configuration overrides.
     * @return distributedlog manager instance.
     * @throws InvalidStreamNameException if the stream name is invalid
     * @throws IOException
     */
protected DistributedLogManager createDistributedLogManager(URI uri, String nameOfLogStream, ClientSharingOption clientSharingOption, Optional<DistributedLogConfiguration> logConfiguration, Optional<DynamicDistributedLogConfiguration> dynamicLogConfiguration) throws InvalidStreamNameException, IOException {
    // Make sure the name is well formed
    validateName(nameOfLogStream);
    DistributedLogConfiguration mergedConfiguration = new DistributedLogConfiguration();
    mergedConfiguration.addConfiguration(conf);
    mergedConfiguration.loadStreamConf(logConfiguration);
    // If dynamic config was not provided, default to a static view of the global configuration.
    DynamicDistributedLogConfiguration dynConf = null;
    if (dynamicLogConfiguration.isPresent()) {
        dynConf = dynamicLogConfiguration.get();
    } else {
        dynConf = ConfUtils.getConstDynConf(mergedConfiguration);
    }
    ZooKeeperClientBuilder writerZKCBuilderForDL = null;
    ZooKeeperClientBuilder readerZKCBuilderForDL = null;
    ZooKeeperClient writerZKCForBK = null;
    ZooKeeperClient readerZKCForBK = null;
    BookKeeperClientBuilder writerBKCBuilder = null;
    BookKeeperClientBuilder readerBKCBuilder = null;
    switch(clientSharingOption) {
        case SharedClients:
            writerZKCBuilderForDL = sharedWriterZKCBuilderForDL;
            readerZKCBuilderForDL = sharedReaderZKCBuilderForDL;
            writerBKCBuilder = sharedWriterBKCBuilder;
            readerBKCBuilder = sharedReaderBKCBuilder;
            break;
        case SharedZKClientPerStreamBKClient:
            writerZKCBuilderForDL = sharedWriterZKCBuilderForDL;
            readerZKCBuilderForDL = sharedReaderZKCBuilderForDL;
            synchronized (this) {
                if (null == this.sharedWriterZKCForBK) {
                    this.sharedWriterZKCBuilderForBK = createBKZKClientBuilder(String.format("bkzk:%s:factory_writer_shared", uri), mergedConfiguration, bkdlConfig.getBkZkServersForWriter(), statsLogger.scope("bkzk_factory_writer_shared"));
                    this.sharedWriterZKCForBK = this.sharedWriterZKCBuilderForBK.build();
                }
                if (null == this.sharedReaderZKCForBK) {
                    if (bkdlConfig.getBkZkServersForWriter().equals(bkdlConfig.getBkZkServersForReader())) {
                        this.sharedReaderZKCBuilderForBK = this.sharedWriterZKCBuilderForBK;
                    } else {
                        this.sharedReaderZKCBuilderForBK = createBKZKClientBuilder(String.format("bkzk:%s:factory_reader_shared", uri), mergedConfiguration, bkdlConfig.getBkZkServersForReader(), statsLogger.scope("bkzk_factory_reader_shared"));
                    }
                    this.sharedReaderZKCForBK = this.sharedReaderZKCBuilderForBK.build();
                }
                writerZKCForBK = this.sharedWriterZKCForBK;
                readerZKCForBK = this.sharedReaderZKCForBK;
            }
            break;
    }
    LedgerAllocator dlmLedgerAlloctor = null;
    PermitManager dlmLogSegmentRollingPermitManager = PermitManager.UNLIMITED_PERMIT_MANAGER;
    if (ClientSharingOption.SharedClients == clientSharingOption) {
        dlmLedgerAlloctor = this.allocator;
        dlmLogSegmentRollingPermitManager = this.logSegmentRollingPermitManager;
    }
    return new BKDistributedLogManager(nameOfLogStream, /* Log Name */
    mergedConfiguration, /* Configuration */
    dynConf, /* Dynamic Configuration */
    uri, /* Namespace */
    writerZKCBuilderForDL, /* ZKC Builder for DL Writer */
    readerZKCBuilderForDL, /* ZKC Builder for DL Reader */
    writerZKCForBK, /* ZKC for BookKeeper for DL Writers */
    readerZKCForBK, /* ZKC for BookKeeper for DL Readers */
    writerBKCBuilder, /* BookKeeper Builder for DL Writers */
    readerBKCBuilder, /* BookKeeper Builder for DL Readers */
    lockFactory, /* Lock Factory */
    writerSegmentMetadataStore, /* Log Segment Metadata Store for DL Writers */
    readerSegmentMetadataStore, /* Log Segment Metadata Store for DL Readers */
    scheduler, /* DL scheduler */
    readAheadExecutor, /* Read Aheader Executor */
    lockStateExecutor, /* Lock State Executor */
    channelFactory, /* Netty Channel Factory */
    requestTimer, /* Request Timer */
    readAheadExceptionsLogger, /* ReadAhead Exceptions Logger */
    clientId, /* Client Id */
    regionId, /* Region Id */
    dlmLedgerAlloctor, /* Ledger Allocator */
    writeLimiter, /* Write Limiter */
    dlmLogSegmentRollingPermitManager, /* Log segment rolling limiter */
    featureProvider.scope("dl"), /* Feature Provider */
    statsLogger, /* Stats Logger */
    perLogStatsLogger);
}

Example 4 with LedgerAllocator

use of com.twitter.distributedlog.bk.LedgerAllocator in project distributedlog by twitter.

the class TestBKLogWriteHandler method testAbortTransactionOnStartLogSegment.

/**
     * Testcase: when write handler encounters exceptions on starting log segment
     * it should abort the transaction and return the ledger to the pool.
     */
@Test(timeout = 60000)
public void testAbortTransactionOnStartLogSegment() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    ensureURICreated(zkc, uri);
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setEnableLedgerAllocatorPool(true);
    confLocal.setLedgerAllocatorPoolCoreSize(1);
    confLocal.setLedgerAllocatorPoolName("test-allocator-pool");
    BKDistributedLogNamespace namespace = (BKDistributedLogNamespace) DistributedLogNamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog("test-stream");
    FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentOnAssignLogSegmentSequenceNumber, FailpointUtils.FailPointActions.FailPointAction_Throw);
    try {
        AsyncLogWriter writer = FutureUtils.result(dlm.openAsyncLogWriter());
        FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
        fail("Should fail opening the writer");
    } catch (IOException ioe) {
    // expected
    } finally {
        FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentOnAssignLogSegmentSequenceNumber);
    }
    LedgerAllocator allocator = namespace.getLedgerAllocator();
    assertTrue(allocator instanceof LedgerAllocatorPool);
    LedgerAllocatorPool allocatorPool = (LedgerAllocatorPool) allocator;
    assertEquals(0, allocatorPool.obtainMapSize());
    AsyncLogWriter writer = FutureUtils.result(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    Utils.close(writer);
}