Examples with StoreDefinition voldemort.store.StoreDefinition used on opensource projects

Example 36 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class ZoneShrinkageEndToEndTest method testAllServersSendingOutSlopsCorrectly.

@Test(timeout = 60000)
public void testAllServersSendingOutSlopsCorrectly() throws InterruptedException {
    final Serializer<ByteArray> slopKeySerializer = new ByteArraySerializer();
    final Serializer<Slop> slopValueSerializer = new SlopSerializer();
    final SlopSerializer slopSerializer = new SlopSerializer();
    StoreDefinition storeDef = storeDefs.get(0);
    TestSocketStoreFactory ssf = new TestSocketStoreFactory();
    Map<Integer, SocketStore> slopStoresCreatedBeforeShrink = new HashMap<Integer, SocketStore>();
    Map<Integer, SocketStore> slopStoresCreatedAfterShrink = new HashMap<Integer, SocketStore>();
    // generate for keys each all servers that will be hosted on each server
    // except itself (2*N*(N-1) keys)
    // Map<Integer slopFinalDestinationNodeId, List<Pair<ByteArray key,
    // Integer hostNodeId>>>
    Map<Integer, List<Pair<ByteArray, Integer>>> serverKeys = new HashMap<Integer, List<Pair<ByteArray, Integer>>>();
    for (Node slopFinalDestinationNode : cluster.getNodes()) {
        serverKeys.put(slopFinalDestinationNode.getId(), new ArrayList<Pair<ByteArray, Integer>>());
    }
    // make socket stores to all servers before shrink
    for (Integer nodeId : vservers.keySet()) {
        SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
        SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
        slopStoresCreatedBeforeShrink.put(nodeId, slopStore);
    }
    for (int i = 0; i < 2; i++) {
        for (Integer slopHostId : vservers.keySet()) {
            SocketStore slopStore = slopStoresCreatedBeforeShrink.get(slopHostId);
            for (Integer destinationNodeId : vservers.keySet()) {
                if (!destinationNodeId.equals(slopHostId)) {
                    ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
                    serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
                    Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
                    slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
                }
            }
        }
    }
    // update metadata
    executeShrinkZone();
    logger.info("-------------------------------");
    logger.info("    CONNECTING SLOP STORES     ");
    logger.info("-------------------------------");
    // make socket stores to all servers after shrink
    for (Integer nodeId : vservers.keySet()) {
        SocketStore slopStore = ssf.createSocketStore(vservers.get(nodeId).getIdentityNode(), "slop");
        SerializingStore.wrap(slopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
        slopStoresCreatedAfterShrink.put(nodeId, slopStore);
    }
    logger.info("-------------------------------");
    logger.info("     CONNECTED SLOP STORES     ");
    logger.info("-------------------------------");
    logger.info("-------------------------------");
    logger.info("         SENDING SLOPS         ");
    logger.info("-------------------------------");
    for (int i = 0; i < 2; i++) {
        for (Integer slopHostId : vservers.keySet()) {
            SocketStore slopStore = slopStoresCreatedAfterShrink.get(slopHostId);
            for (Integer destinationNodeId : vservers.keySet()) {
                if (!destinationNodeId.equals(slopHostId)) {
                    ByteArray key = generateRandomKey(cluster, destinationNodeId, storeDef.getReplicationFactor());
                    serverKeys.get(destinationNodeId).add(new Pair<ByteArray, Integer>(key, slopHostId));
                    Slop slop = new Slop(storeDef.getName(), Slop.Operation.PUT, key.get(), key.get(), destinationNodeId, new Date());
                    slopStore.put(slop.makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop), new VectorClock()), null);
                }
            }
        }
    }
    logger.info("-------------------------------");
    logger.info("           SENT SLOPS          ");
    logger.info("-------------------------------");
    ServerTestUtils.waitForSlopDrain(vservers, 30000L);
    // verify all proper slops is processed properly (arrived or dropped)
    boolean hasError = false;
    int goodCount = 0;
    int errorCount = 0;
    for (Integer nodeId : serverKeys.keySet()) {
        VoldemortServer vs = vservers.get(nodeId);
        Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getStorageEngine(storeDef.getName());
        List<Pair<ByteArray, Integer>> keySet = serverKeys.get(nodeId);
        for (Pair<ByteArray, Integer> keyHostIdPair : keySet) {
            ByteArray key = keyHostIdPair.getFirst();
            Integer hostId = keyHostIdPair.getSecond();
            Integer nodeZoneId = cluster.getNodeById(nodeId).getZoneId();
            List<Versioned<byte[]>> result = store.get(key, null);
            if (cluster.getNodeById(nodeId).getZoneId() == droppingZoneId) {
                if (!result.isEmpty()) {
                    logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should be gone but exists\n", key.toString(), nodeId, nodeZoneId, hostId));
                    hasError = true;
                    errorCount++;
                } else {
                    goodCount++;
                }
            } else {
                if (result.isEmpty()) {
                    logger.error(String.format("Key %s for Node %d (zone %d) slopped on Node %d should exist but not\n", key.toString(), nodeId, nodeZoneId, hostId));
                    hasError = true;
                    errorCount++;
                } else {
                    goodCount++;
                }
            }
        }
    }
    logger.info(String.format("Good keys count: %d; Error keys count: %d", goodCount, errorCount));
    Assert.assertFalse("Error Occurred BAD:" + errorCount + "; GOOD: " + goodCount + ". Check log.", hasError);
}

Example 37 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class ZonedRebalanceNonContiguousZonesTest method testShuffleZ1Z3AndShuffleAgain.

@Test(timeout = 600000)
public void testShuffleZ1Z3AndShuffleAgain() throws Exception {
    logger.info("Starting testShuffleZZAndShuffleAgain");
    // Hacky work around of TOCTOU bind Exception issues. Each test that invokes this method brings servers
    // up & down on the same ports. The OS seems to need a rest between subsequent tests...
    Thread.sleep(TimeUnit.SECONDS.toMillis(5));
    Cluster interimCluster = RebalanceUtils.getInterimCluster(z1z3Current, z1z3Shuffle);
    // start all the servers
    List<Integer> serverList = new ArrayList<Integer>(interimCluster.getNodeIds());
    Map<String, String> configProps = new HashMap<String, String>();
    configProps.put("admin.max.threads", "5");
    interimCluster = startServers(interimCluster, z1z3StoresXml, serverList, configProps);
    // Populate cluster with data
    for (StoreDefinition storeDef : z1z3Stores) {
        populateData(z1z3Current, storeDef);
    }
    String bootstrapUrl = getBootstrapUrl(interimCluster, 3);
    // Shuffle cluster
    ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, z1z3Shuffle, z1z3Stores);
    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, serverList);
    checkConsistentMetadata(z1z3Shuffle, serverList);
    // Now, go from shuffled state, back to the original to confirm subsequent rebalances can be invoked.
    rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, z1z3Current, z1z3Stores);
    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, serverList);
    checkConsistentMetadata(z1z3Current, serverList);
    // Done.
    stopServer(serverList);
}

Example 38 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class ReplaceNodeTest method testNodeDownReplacement.

@Test
public void testNodeDownReplacement() throws Exception {
    final int REPLACEMENT_NODE = 0;
    // This is to simulate the case where a machine failed but hard disk was intact
    // In this case we will move the hard disk to another machine, edit the cluster.xml
    // to point to this host and start this machine.
    // The case is simulated by remembering the cluster of node A.
    // Replace this node with node B. Now create a new server with cluster of
    // node A ( this is to simulate the cluster.xml edit) and data directory of B
    // ( this is to simulate the hard disk move). Now try replacing the node B with
    // newly created node after shutting down the node B.
    Cluster cluster = originalServers[nodeToBeReplaced].getMetadataStore().getCluster();
    List<StoreDefinition> storeDefs = originalServers[nodeToBeReplaced].getMetadataStore().getStoreDefList();
    Node node = originalServers[nodeToBeReplaced].getIdentityNode();
    // Verify the node down scenario first
    final boolean DO_RESTORE = false;
    final boolean STOP_OLD_NODE = true;
    verifyNodeReplacement(nodeToBeReplaced, otherServers, REPLACEMENT_NODE, STOP_OLD_NODE, DO_RESTORE);
    // Now the replacement node is part of the original cluster.
    // Stop the replacement node, assume you moved the hard disk
    // to a new node ( This is done by starting another node)
    // that points to the data directory of the replacement node.
    String baseDirPath = otherServers[REPLACEMENT_NODE].getVoldemortConfig().getVoldemortHome();
    // Using ServerTestUtils stopVoldemortServer also deletes the data
    // directory so using the stop, to simulate the hard disk move.
    otherServers[REPLACEMENT_NODE].stop();
    VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs(true, node.getId(), baseDirPath, cluster, storeDefs, new Properties());
    Assert.assertTrue(config.isSlopEnabled());
    Assert.assertTrue(config.isSlopPusherJobEnabled());
    Assert.assertTrue(config.getAutoPurgeDeadSlops());
    config.setSlopFrequencyMs(8000L);
    config.setEnableNodeIdDetection(this.autoDetectNodeId);
    if (this.autoDetectNodeId) {
        config.setNodeIdImplementation(new NodeIdHostMatcher(nodeToBeReplaced));
    }
    // Though baseDir is used as part of Config, TestUtils, always appends
    // it with node- (nodeId). So forcefully overwriting it here to point to
    // the old directory.
    config.setVoldemortHome(baseDirPath);
    SocketStoreFactory ssf = new TestSocketStoreFactory();
    VoldemortServer hardDiskMovedServer = ServerTestUtils.startVoldemortServer(ssf, config, cluster);
    ssf.close();
    otherServers[REPLACEMENT_NODE] = hardDiskMovedServer;
    final boolean SKIP_RESTORE = true;
    final boolean DONOT_STOP_OLD_NODE = true;
    verifyNodeReplacement(nodeToBeReplaced, otherServers, REPLACEMENT_NODE, DONOT_STOP_OLD_NODE, SKIP_RESTORE);
}

Example 39 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class VerifyOrAddStoreTest method verifyStoreDoesNotExist.

private void verifyStoreDoesNotExist() {
    StoreDefinition retrieved = adminClient.metadataMgmtOps.getStoreDefinition(newStoreName);
    assertNull("Store should not be created", retrieved);
    for (Integer nodeId : cluster.getNodeIds()) {
        retrieved = retrieveStoreOnNode(newStoreName, nodeId);
        assertNull("Store should not be created", retrieved);
        Long quota = getQuotaForNode(newStoreName, QuotaType.STORAGE_SPACE, nodeId);
        assertNull("Default quota should not exist", quota);
    }
}

Example 40 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class VerifyOrAddStoreTest method verifyMisMatchFails.

@Test
public void verifyMisMatchFails() throws Exception {
    StoreDefinition incompatibleDef = getIncompatibleStoreDef();
    // Add the incompatible store definition to Node 0
    adminClient.storeMgmtOps.addStore(incompatibleDef, FAILED_NODE_ID);
    // Adding the store with different definition should fail.
    try {
        adminClient.storeMgmtOps.verifyOrAddStore(newStoreDef, PROCESS_NAME, false, service);
        Assert.fail("Non existent store create with flag disabled should have thrown error");
    } catch (VoldemortException ex) {
    // Pass
    }
    for (Integer nodeId : cluster.getNodeIds()) {
        StoreDefinition retrieved = retrieveStoreOnNode(newStoreName, nodeId);
        if (nodeId == FAILED_NODE_ID) {
            assertEquals("mismatched store def should be left intact", incompatibleDef, retrieved);
        } else {
            assertNull("Store should not exist in this node", retrieved);
        }
    }
}