Examples with SettableFeatureProvider org.apache.bookkeeper.feature.SettableFeatureProvider used on opensource projects

Example 1 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project bookkeeper by apache.

the class NamespaceBuilder method build.

/**
 * Build the namespace.
 *
 * @return the namespace instance.
 * @throws IllegalArgumentException when there is illegal argument provided in the builder
 * @throws NullPointerException when there is null argument provided in the builder
 * @throws IOException when fail to build the backend
 */
public Namespace build() throws IllegalArgumentException, NullPointerException, IOException {
    // Check arguments
    checkNotNull(_conf, "No DistributedLog Configuration.");
    checkNotNull(_uri, "No DistributedLog URI");
    // validate the configuration
    _conf.validate();
    if (null == _dynConf) {
        _dynConf = ConfUtils.getConstDynConf(_conf);
    }
    // retrieve the namespace driver
    NamespaceDriver driver = NamespaceDriverManager.getDriver(_uri);
    URI normalizedUri = DLUtils.normalizeURI(_uri);
    // build the feature provider
    FeatureProvider featureProvider;
    if (null == _featureProvider) {
        featureProvider = new SettableFeatureProvider("", 0);
        logger.info("No feature provider is set. All features are disabled now.");
    } else {
        featureProvider = _featureProvider;
    }
    // build the failure injector
    AsyncFailureInjector failureInjector = AsyncRandomFailureInjector.newBuilder().injectDelays(_conf.getEIInjectReadAheadDelay(), _conf.getEIInjectReadAheadDelayPercent(), _conf.getEIInjectMaxReadAheadDelayMs()).injectErrors(false, 10).injectStops(_conf.getEIInjectReadAheadStall(), 10).injectCorruption(_conf.getEIInjectReadAheadBrokenEntries()).build();
    // normalize the per log stats logger
    StatsLogger perLogStatsLogger = normalizePerLogStatsLogger(_statsLogger, _perLogStatsLogger, _conf);
    // build the scheduler
    OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("DLM-" + normalizedUri.getPath()).numThreads(_conf.getNumWorkerThreads()).build();
    // initialize the namespace driver
    driver.initialize(_conf, _dynConf, normalizedUri, scheduler, featureProvider, failureInjector, _statsLogger, perLogStatsLogger, DLUtils.normalizeClientId(_clientId), _regionId);
    // initialize the write limiter
    PermitLimiter writeLimiter;
    if (_conf.getGlobalOutstandingWriteLimit() < 0) {
        writeLimiter = PermitLimiter.NULL_PERMIT_LIMITER;
    } else {
        Feature disableWriteLimitFeature = featureProvider.getFeature(CoreFeatureKeys.DISABLE_WRITE_LIMIT.name().toLowerCase());
        writeLimiter = new SimplePermitLimiter(_conf.getOutstandingWriteLimitDarkmode(), _conf.getGlobalOutstandingWriteLimit(), _statsLogger.scope("writeLimiter"), true, /* singleton */
        disableWriteLimitFeature);
    }
    return new BKDistributedLogNamespace(_conf, normalizedUri, driver, scheduler, featureProvider, writeLimiter, failureInjector, _statsLogger, perLogStatsLogger, DLUtils.normalizeClientId(_clientId), _regionId);
}

Example 2 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project bookkeeper by apache.

the class DLMTestUtil method injectLogSegmentWithGivenLogSegmentSeqNo.

public static void injectLogSegmentWithGivenLogSegmentSeqNo(DistributedLogManager manager, DistributedLogConfiguration conf, long logSegmentSeqNo, long startTxID, boolean writeEntries, long segmentSize, boolean completeLogSegment) throws Exception {
    BKDistributedLogManager dlm = (BKDistributedLogManager) manager;
    BKLogWriteHandler writeHandler = dlm.createWriteHandler(false);
    Utils.ioResult(writeHandler.lockHandler());
    // Start a log segment with a given ledger seq number.
    BookKeeperClient bkc = getBookKeeperClient(dlm);
    LedgerHandle lh = bkc.get().createLedger(conf.getEnsembleSize(), conf.getWriteQuorumSize(), conf.getAckQuorumSize(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes());
    String inprogressZnodeName = writeHandler.inprogressZNodeName(lh.getId(), startTxID, logSegmentSeqNo);
    String znodePath = writeHandler.inprogressZNode(lh.getId(), startTxID, logSegmentSeqNo);
    int logSegmentMetadataVersion = conf.getDLLedgerMetadataLayoutVersion();
    LogSegmentMetadata l = new LogSegmentMetadata.LogSegmentMetadataBuilder(znodePath, logSegmentMetadataVersion, lh.getId(), startTxID).setLogSegmentSequenceNo(logSegmentSeqNo).setEnvelopeEntries(LogSegmentMetadata.supportsEnvelopedEntries(logSegmentMetadataVersion)).build();
    l.write(getZooKeeperClient(dlm));
    writeHandler.maxTxId.update(Version.ANY, startTxID);
    writeHandler.addLogSegmentToCache(inprogressZnodeName, l);
    BKLogSegmentWriter writer = new BKLogSegmentWriter(writeHandler.getFullyQualifiedName(), inprogressZnodeName, conf, conf.getDLLedgerMetadataLayoutVersion(), new BKLogSegmentEntryWriter(lh), writeHandler.lock, startTxID, logSegmentSeqNo, writeHandler.scheduler, writeHandler.statsLogger, writeHandler.statsLogger, writeHandler.alertStatsLogger, PermitLimiter.NULL_PERMIT_LIMITER, new SettableFeatureProvider("", 0), ConfUtils.getConstDynConf(conf));
    if (writeEntries) {
        long txid = startTxID;
        for (long j = 1; j <= segmentSize; j++) {
            writer.write(DLMTestUtil.getLogRecordInstance(txid++));
        }
        Utils.ioResult(writer.flushAndCommit());
    }
    if (completeLogSegment) {
        Utils.ioResult(writeHandler.completeAndCloseLogSegment(writer));
    }
    Utils.ioResult(writeHandler.unlockHandler());
}

Example 3 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project bookkeeper by apache.

the class TestDistributedLogBase method createNewDLM.

public BKDistributedLogManager createNewDLM(DistributedLogConfiguration conf, String name, PermitLimiter writeLimiter) throws Exception {
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    final Namespace namespace = NamespaceBuilder.newBuilder().uri(uri).conf(conf).build();
    final OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("test-scheduler").build();
    AsyncCloseable resourcesCloseable = new AsyncCloseable() {

        @Override
        public CompletableFuture<Void> asyncClose() {
            LOG.info("Shutting down the scheduler");
            SchedulerUtils.shutdownScheduler(scheduler, 1, TimeUnit.SECONDS);
            LOG.info("Shut down the scheduler");
            LOG.info("Closing the namespace");
            namespace.close();
            LOG.info("Closed the namespace");
            return FutureUtils.Void();
        }
    };
    AsyncFailureInjector failureInjector = AsyncRandomFailureInjector.newBuilder().injectDelays(conf.getEIInjectReadAheadDelay(), conf.getEIInjectReadAheadDelayPercent(), conf.getEIInjectMaxReadAheadDelayMs()).injectErrors(false, 10).injectStops(conf.getEIInjectReadAheadStall(), 10).injectCorruption(conf.getEIInjectReadAheadBrokenEntries()).build();
    return new BKDistributedLogManager(name, conf, ConfUtils.getConstDynConf(conf), uri, namespace.getNamespaceDriver(), new LogSegmentMetadataCache(conf, Ticker.systemTicker()), scheduler, DistributedLogConstants.UNKNOWN_CLIENT_ID, DistributedLogConstants.LOCAL_REGION_ID, writeLimiter, new SettableFeatureProvider("", 0), failureInjector, NullStatsLogger.INSTANCE, NullStatsLogger.INSTANCE, Optional.of(resourcesCloseable));
}

Example 4 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project bookkeeper by apache.

the class TestRegionAwareEnsemblePlacementPolicy method testNewEnsembleWithThreeRegionsWithDisable.

@Test
public void testNewEnsembleWithThreeRegionsWithDisable() throws Exception {
    FeatureProvider featureProvider = new SettableFeatureProvider("", 0);
    repp.uninitalize();
    repp = new RegionAwareEnsemblePlacementPolicy();
    conf.setProperty(REPP_DISALLOW_BOOKIE_PLACEMENT_IN_REGION_FEATURE_NAME, "disallowBookies");
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, featureProvider, NullStatsLogger.INSTANCE);
    BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.2", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.3", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.4", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.5", 3181);
    BookieSocketAddress addr5 = new BookieSocketAddress("127.0.0.6", 3181);
    BookieSocketAddress addr6 = new BookieSocketAddress("127.0.0.7", 3181);
    BookieSocketAddress addr7 = new BookieSocketAddress("127.0.0.8", 3181);
    BookieSocketAddress addr8 = new BookieSocketAddress("127.0.0.9", 3181);
    BookieSocketAddress addr9 = new BookieSocketAddress("127.0.0.10", 3181);
    BookieSocketAddress addr10 = new BookieSocketAddress("127.0.0.11", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region2/r1");
    StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/r2");
    StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region2/r3");
    StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region3/r4");
    StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/region1/r11");
    StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/region1/r12");
    StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/region2/r13");
    StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/region3/r14");
    StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/region2/r23");
    StaticDNSResolver.addNodeToRack(addr10.getHostName(), "/region1/r24");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    addrs.add(addr5);
    addrs.add(addr6);
    addrs.add(addr7);
    addrs.add(addr8);
    addrs.add(addr9);
    addrs.add(addr10);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    try {
        ((SettableFeature) featureProvider.scope("region1").getFeature("disallowBookies")).set(true);
        ArrayList<BookieSocketAddress> ensemble = repp.newEnsemble(6, 6, 4, null, new HashSet<BookieSocketAddress>());
        assertEquals(2, getNumRegionsInEnsemble(ensemble));
        assert (ensemble.contains(addr1));
        assert (ensemble.contains(addr3));
        assert (ensemble.contains(addr4));
        assert (ensemble.contains(addr7));
        assert (ensemble.contains(addr8));
        assert (ensemble.contains(addr9));
        assert (ensemble.size() == 6);
    } catch (BKNotEnoughBookiesException bnebe) {
        fail("Should not get not enough bookies exception even there is only one rack.");
    }
    try {
        ((SettableFeature) featureProvider.scope("region2").getFeature("disallowBookies")).set(true);
        ArrayList<BookieSocketAddress> ensemble = repp.newEnsemble(6, 6, 4, null, new HashSet<BookieSocketAddress>());
        fail("Should get not enough bookies exception even there is only one region with insufficient bookies.");
    } catch (BKNotEnoughBookiesException bnebe) {
    // Expected
    }
    try {
        ((SettableFeature) featureProvider.scope("region2").getFeature("disallowBookies")).set(false);
        ArrayList<BookieSocketAddress> ensemble = repp.newEnsemble(6, 6, 4, null, new HashSet<BookieSocketAddress>());
        assert (ensemble.contains(addr1));
        assert (ensemble.contains(addr3));
        assert (ensemble.contains(addr4));
        assert (ensemble.contains(addr7));
        assert (ensemble.contains(addr8));
        assert (ensemble.contains(addr9));
        assert (ensemble.size() == 6);
        assertEquals(2, getNumRegionsInEnsemble(ensemble));
    } catch (BKNotEnoughBookiesException bnebe) {
        fail("Should not get not enough bookies exception even there is only one rack.");
    }
}

Example 5 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project bookkeeper by apache.

the class TestRegionAwareEnsemblePlacementPolicy method testEnsembleDurabilityDisabledInternal.

public void testEnsembleDurabilityDisabledInternal(int minDurability, boolean disableDurability) throws Exception {
    repp.uninitalize();
    repp = new RegionAwareEnsemblePlacementPolicy();
    conf.setProperty(REPP_REGIONS_TO_WRITE, "region1;region2;region3");
    conf.setProperty(REPP_MINIMUM_REGIONS_FOR_DURABILITY, minDurability);
    FeatureProvider featureProvider = new SettableFeatureProvider("", 0);
    if (minDurability <= 1) {
        conf.setProperty(REPP_ENABLE_DURABILITY_ENFORCEMENT_IN_REPLACE, false);
    } else {
        conf.setProperty(REPP_ENABLE_DURABILITY_ENFORCEMENT_IN_REPLACE, true);
    }
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, featureProvider, NullStatsLogger.INSTANCE);
    BookieSocketAddress addr1 = new BookieSocketAddress("127.1.0.2", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.1.0.3", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.1.0.4", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.1.0.5", 3181);
    BookieSocketAddress addr5 = new BookieSocketAddress("127.1.0.6", 3181);
    BookieSocketAddress addr6 = new BookieSocketAddress("127.1.0.7", 3181);
    BookieSocketAddress addr7 = new BookieSocketAddress("127.1.0.8", 3181);
    BookieSocketAddress addr8 = new BookieSocketAddress("127.1.0.9", 3181);
    BookieSocketAddress addr9 = new BookieSocketAddress("127.1.0.10", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region1/r1");
    StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/r2");
    StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region1/r3");
    StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region1/r4");
    StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/region1/r11");
    StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/region1/r12");
    StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/region1/r13");
    StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/region1/r14");
    StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/region1/r23");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    addrs.add(addr5);
    addrs.add(addr6);
    addrs.add(addr7);
    addrs.add(addr8);
    addrs.add(addr9);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    if (disableDurability) {
        ((SettableFeature) featureProvider.getFeature(BookKeeperConstants.FEATURE_REPP_DISABLE_DURABILITY_ENFORCEMENT)).set(true);
    }
    ArrayList<BookieSocketAddress> ensemble;
    try {
        ensemble = repp.newEnsemble(6, 6, 4, null, new HashSet<BookieSocketAddress>());
        assert (ensemble.size() == 6);
    } catch (BKNotEnoughBookiesException bnebe) {
        LOG.error("BKNotEnoughBookiesException", bnebe);
        fail("Should not get not enough bookies exception even there is only one rack.");
        throw bnebe;
    }
    Set<BookieSocketAddress> excludedAddrs = new HashSet<BookieSocketAddress>();
    try {
        repp.replaceBookie(6, 6, 4, null, new HashSet<>(ensemble), addr4, excludedAddrs);
    } catch (BKNotEnoughBookiesException bnebe) {
        fail("Should not get not enough bookies exception even there is only one rack.");
    }
}