Examples with SettableFeatureProvider org.apache.bookkeeper.feature.SettableFeatureProvider used on opensource projects

Example 11 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project bookkeeper by apache.

the class TestRegionAwareEnsemblePlacementPolicy method testEnsembleWithThreeRegionsReplaceInternal.

public void testEnsembleWithThreeRegionsReplaceInternal(int minDurability, boolean disableDurability, boolean disableOneRegion) throws Exception {
    repp.uninitalize();
    repp = new RegionAwareEnsemblePlacementPolicy();
    conf.setProperty(REPP_REGIONS_TO_WRITE, "region1;region2;region3");
    conf.setProperty(REPP_MINIMUM_REGIONS_FOR_DURABILITY, minDurability);
    FeatureProvider featureProvider = new SettableFeatureProvider("", 0);
    if (minDurability <= 1) {
        conf.setProperty(REPP_ENABLE_DURABILITY_ENFORCEMENT_IN_REPLACE, false);
    } else {
        conf.setProperty(REPP_ENABLE_DURABILITY_ENFORCEMENT_IN_REPLACE, true);
    }
    conf.setProperty(REPP_DISALLOW_BOOKIE_PLACEMENT_IN_REGION_FEATURE_NAME, "disallowBookies");
    repp.initialize(conf, Optional.<DNSToSwitchMapping>empty(), timer, featureProvider, NullStatsLogger.INSTANCE);
    BookieSocketAddress addr1 = new BookieSocketAddress("127.1.0.2", 3181);
    BookieSocketAddress addr2 = new BookieSocketAddress("127.1.0.3", 3181);
    BookieSocketAddress addr3 = new BookieSocketAddress("127.1.0.4", 3181);
    BookieSocketAddress addr4 = new BookieSocketAddress("127.1.0.5", 3181);
    BookieSocketAddress addr5 = new BookieSocketAddress("127.1.0.6", 3181);
    BookieSocketAddress addr6 = new BookieSocketAddress("127.1.0.7", 3181);
    BookieSocketAddress addr7 = new BookieSocketAddress("127.1.0.8", 3181);
    BookieSocketAddress addr8 = new BookieSocketAddress("127.1.0.9", 3181);
    BookieSocketAddress addr9 = new BookieSocketAddress("127.1.0.10", 3181);
    // update dns mapping
    StaticDNSResolver.addNodeToRack(addr1.getHostName(), "/region1/r1");
    StaticDNSResolver.addNodeToRack(addr2.getHostName(), "/region1/r2");
    StaticDNSResolver.addNodeToRack(addr3.getHostName(), "/region1/r3");
    StaticDNSResolver.addNodeToRack(addr4.getHostName(), "/region2/r4");
    StaticDNSResolver.addNodeToRack(addr5.getHostName(), "/region2/r11");
    StaticDNSResolver.addNodeToRack(addr6.getHostName(), "/region2/r12");
    StaticDNSResolver.addNodeToRack(addr7.getHostName(), "/region3/r13");
    StaticDNSResolver.addNodeToRack(addr8.getHostName(), "/region3/r14");
    StaticDNSResolver.addNodeToRack(addr9.getHostName(), "/region3/r23");
    // Update cluster
    Set<BookieSocketAddress> addrs = new HashSet<BookieSocketAddress>();
    addrs.add(addr1);
    addrs.add(addr2);
    addrs.add(addr3);
    addrs.add(addr4);
    addrs.add(addr5);
    addrs.add(addr6);
    addrs.add(addr7);
    addrs.add(addr8);
    addrs.add(addr9);
    repp.onClusterChanged(addrs, new HashSet<BookieSocketAddress>());
    SettableFeature disableDurabilityFeature = (SettableFeature) featureProvider.getFeature(BookKeeperConstants.FEATURE_REPP_DISABLE_DURABILITY_ENFORCEMENT);
    if (disableDurability) {
        disableDurabilityFeature.set(true);
    }
    int ackQuorum = 4;
    if (minDurability > 2) {
        ackQuorum = 5;
    }
    ArrayList<BookieSocketAddress> ensemble;
    try {
        ensemble = repp.newEnsemble(6, 6, ackQuorum, null, new HashSet<BookieSocketAddress>());
        assert (ensemble.size() == 6);
        assertEquals(3, getNumRegionsInEnsemble(ensemble));
    } catch (BKNotEnoughBookiesException bnebe) {
        LOG.error("BKNotEnoughBookiesException", bnebe);
        fail("Should not get not enough bookies exception even there is only one rack.");
        throw bnebe;
    }
    if (disableOneRegion) {
        ((SettableFeature) featureProvider.scope("region2").getFeature("disallowBookies")).set(true);
        Set<BookieSocketAddress> region2Bookies = new HashSet<BookieSocketAddress>();
        region2Bookies.add(addr4);
        region2Bookies.add(addr5);
        region2Bookies.add(addr6);
        Set<BookieSocketAddress> region1And3Bookies = new HashSet<BookieSocketAddress>(addrs);
        region1And3Bookies.removeAll(region2Bookies);
        Set<BookieSocketAddress> excludedAddrs = new HashSet<BookieSocketAddress>();
        for (BookieSocketAddress addr : region2Bookies) {
            if (ensemble.contains(addr)) {
                BookieSocketAddress replacedBookie = repp.replaceBookie(6, 6, ackQuorum, null, new HashSet<>(ensemble), addr, excludedAddrs);
                ensemble.remove(addr);
                ensemble.add(replacedBookie);
            }
        }
        assertEquals(2, getNumRegionsInEnsemble(ensemble));
        assertTrue(ensemble.containsAll(region1And3Bookies));
    } else {
        BookieSocketAddress bookieToReplace;
        BookieSocketAddress replacedBookieExpected;
        if (ensemble.contains(addr4)) {
            bookieToReplace = addr4;
            if (ensemble.contains(addr5)) {
                replacedBookieExpected = addr6;
            } else {
                replacedBookieExpected = addr5;
            }
        } else {
            replacedBookieExpected = addr4;
            bookieToReplace = addr5;
        }
        Set<BookieSocketAddress> excludedAddrs = new HashSet<BookieSocketAddress>();
        try {
            BookieSocketAddress replacedBookie = repp.replaceBookie(6, 6, ackQuorum, null, new HashSet<>(ensemble), bookieToReplace, excludedAddrs);
            assert (replacedBookie.equals(replacedBookieExpected));
            assertEquals(3, getNumRegionsInEnsemble(ensemble));
        } catch (BKNotEnoughBookiesException bnebe) {
            fail("Should not get not enough bookies exception even there is only one rack.");
        }
        excludedAddrs.add(replacedBookieExpected);
        try {
            BookieSocketAddress replacedBookie = repp.replaceBookie(6, 6, ackQuorum, null, new HashSet<>(ensemble), bookieToReplace, excludedAddrs);
            if (minDurability > 1 && !disableDurabilityFeature.isAvailable()) {
                fail("Should throw BKNotEnoughBookiesException when there is not enough bookies");
            }
        } catch (BKNotEnoughBookiesException bnebe) {
            if (minDurability <= 1 || disableDurabilityFeature.isAvailable()) {
                fail("Should not throw BKNotEnoughBookiesException when there is not enough bookies");
            }
        }
    }
}

Example 12 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project bookkeeper by apache.

the class TestDisableEnsembleChange method testRetrySlowBookie.

@Test
public void testRetrySlowBookie() throws Exception {
    final int readTimeout = 2;
    ClientConfiguration conf = new ClientConfiguration();
    conf.setReadEntryTimeout(readTimeout).setAddEntryTimeout(readTimeout).setDelayEnsembleChange(false).setDisableEnsembleChangeFeatureName(FEATURE_DISABLE_ENSEMBLE_CHANGE).setZkServers(zkUtil.getZooKeeperConnectString());
    SettableFeatureProvider featureProvider = new SettableFeatureProvider("test", 0);
    BookKeeper bkc = BookKeeper.forConfig(conf).featureProvider(featureProvider).build();
    SettableFeature disableEnsembleChangeFeature = featureProvider.getFeature(FEATURE_DISABLE_ENSEMBLE_CHANGE);
    disableEnsembleChangeFeature.set(true);
    LedgerHandle lh = bkc.createLedger(4, 4, 4, BookKeeper.DigestType.CRC32, new byte[] {});
    byte[] entry = "testRetryFailureBookie".getBytes();
    for (int i = 0; i < 10; i++) {
        lh.addEntry(entry);
    }
    List<BookieSocketAddress> curEns = lh.getLedgerMetadata().currentEnsemble;
    final CountDownLatch wakeupLatch = new CountDownLatch(1);
    final CountDownLatch suspendLatch = new CountDownLatch(1);
    sleepBookie(curEns.get(2), wakeupLatch, suspendLatch);
    suspendLatch.await();
    final AtomicInteger res = new AtomicInteger(0xdeadbeef);
    final CountDownLatch addLatch = new CountDownLatch(1);
    AsyncCallback.AddCallback cb = new AsyncCallback.AddCallback() {

        @Override
        public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
            logger.info("Add entry {} completed : rc {}.", entryId, rc);
            res.set(rc);
            addLatch.countDown();
        }
    };
    lh.asyncAddEntry(entry, cb, null);
    assertFalse("Add entry operation should not complete.", addLatch.await(1000, TimeUnit.MILLISECONDS));
    assertEquals(res.get(), 0xdeadbeef);
    // wait until read timeout
    assertFalse("Add entry operation should not complete even timeout.", addLatch.await(readTimeout, TimeUnit.SECONDS));
    assertEquals(res.get(), 0xdeadbeef);
    // wait one more read timeout, to ensure we resend multiple retries
    // to ensure it works correctly
    assertFalse("Add entry operation should not complete even timeout.", addLatch.await(readTimeout, TimeUnit.SECONDS));
    assertEquals(res.get(), 0xdeadbeef);
    // wakeup the sleep bookie
    wakeupLatch.countDown();
    assertTrue("Add entry operation should complete at this point.", addLatch.await(10000, TimeUnit.MILLISECONDS));
    assertEquals(res.get(), BKException.Code.OK);
}

Example 13 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project distributedlog by twitter.

the class DLMTestUtil method createNewBKDLM.

static BKLogPartitionWriteHandlerAndClients createNewBKDLM(DistributedLogConfiguration conf, String logName, int zkPort) throws Exception {
    URI uri = createDLMURI(zkPort, "/" + logName);
    ZooKeeperClientBuilder zkcBuilder = TestZooKeeperClientBuilder.newBuilder(conf).name(String.format("dlzk:%s:handler_dedicated", logName)).uri(uri);
    ZooKeeperClient zkClient = zkcBuilder.build();
    try {
        zkClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (KeeperException.NodeExistsException nee) {
    // ignore
    }
    // resolve uri
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkClient, uri);
    BKDLConfig.propagateConfiguration(bkdlConfig, conf);
    BookKeeperClientBuilder bkcBuilder = BookKeeperClientBuilder.newBuilder().dlConfig(conf).name(String.format("bk:%s:handler_dedicated", logName)).zkServers(bkdlConfig.getBkZkServersForWriter()).ledgersPath(bkdlConfig.getBkLedgersPath()).statsLogger(NullStatsLogger.INSTANCE);
    BKDistributedLogManager bkdlm = new BKDistributedLogManager(logName, conf, uri, zkcBuilder, zkcBuilder, zkClient, zkClient, bkcBuilder, bkcBuilder, new SettableFeatureProvider("", 0), PermitLimiter.NULL_PERMIT_LIMITER, NullStatsLogger.INSTANCE);
    BKLogWriteHandler writeHandler = bkdlm.createWriteHandler(true);
    return new BKLogPartitionWriteHandlerAndClients(writeHandler, zkClient, bkcBuilder.build());
}

Example 14 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project distributedlog by twitter.

the class DistributedLogNamespaceBuilder method build.

/**
 * Build the namespace.
 *
 * @return the namespace instance.
 * @throws IllegalArgumentException when there is illegal argument provided in the builder
 * @throws NullPointerException when there is null argument provided in the builder
 * @throws IOException when fail to build the backend
 */
public DistributedLogNamespace build() throws IllegalArgumentException, NullPointerException, IOException {
    // Check arguments
    Preconditions.checkNotNull(_conf, "No DistributedLog Configuration.");
    Preconditions.checkNotNull(_uri, "No DistributedLog URI");
    // Validate the uri and load the backend according to scheme
    String scheme = _uri.getScheme();
    Preconditions.checkNotNull(scheme, "Invalid DistributedLog URI : " + _uri);
    String[] schemeParts = StringUtils.split(scheme, '-');
    Preconditions.checkArgument(schemeParts.length > 0, "Invalid distributedlog scheme found : " + _uri);
    Preconditions.checkArgument(Objects.equal(DistributedLogConstants.SCHEME_PREFIX, schemeParts[0].toLowerCase()), "Unknown distributedlog scheme found : " + _uri);
    // if we are going to support other backends : e.g. 'distributedlog-mem:'.
    if (schemeParts.length > 1) {
        String backendProvider = schemeParts[1];
        Preconditions.checkArgument(Objects.equal(DistributedLogConstants.BACKEND_BK, backendProvider.toLowerCase()), "Backend '" + backendProvider + "' is not supported yet.");
    }
    // Built the feature provider
    FeatureProvider featureProvider;
    if (null == _featureProvider) {
        featureProvider = new SettableFeatureProvider("", 0);
        logger.info("No feature provider is set. All features are disabled now.");
    } else {
        featureProvider = _featureProvider;
    }
    URI bkUri;
    try {
        bkUri = new URI(// remove backend info from bookkeeper backend
        schemeParts[0], _uri.getAuthority(), _uri.getPath(), _uri.getQuery(), _uri.getFragment());
    } catch (URISyntaxException e) {
        throw new IllegalArgumentException("Invalid distributedlog uri found : " + _uri, e);
    }
    return BKDistributedLogNamespace.newBuilder().conf(_conf).uri(bkUri).statsLogger(_statsLogger).featureProvider(featureProvider).clientId(_clientId).regionId(_regionId).build();
}

Example 15 with SettableFeatureProvider

use of org.apache.bookkeeper.feature.SettableFeatureProvider in project distributedlog by twitter.

the class TestDistributedLogBase method createNewDLM.

public BKDistributedLogManager createNewDLM(DistributedLogConfiguration conf, String name) throws Exception {
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    return new BKDistributedLogManager(name, conf, uri, null, null, null, null, null, null, new SettableFeatureProvider("", 0), PermitLimiter.NULL_PERMIT_LIMITER, NullStatsLogger.INSTANCE);
}