use of voldemort.routing.RoutingStrategyFactory in project voldemort by voldemort.
the class HdfsFetcher method getPartitionsForCurrentNode.
/**
* @return the set of partitions which need to be served (and thus fetched) by the current node.
*/
private Set<Integer> getPartitionsForCurrentNode(MetadataStore metadataStore, String storeName) {
Set<Integer> partitions = Sets.newHashSet();
StoreDefinition storeDefinition = metadataStore.getStoreDef(storeName);
Cluster cluster = metadataStore.getCluster();
List<Integer> partitionsMasteredByCurrentNode = cluster.getNodeById(metadataStore.getNodeId()).getPartitionIds();
RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
// For each partition in the cluster, determine if it needs to be served by the current node
for (int partitionId = 0; partitionId < cluster.getNumberOfPartitions(); partitionId++) {
boolean partitionIdIsServedByCurrentNode = false;
for (Integer replicatingPartition : routingStrategy.getReplicatingPartitionList(partitionId)) {
if (partitionsMasteredByCurrentNode.contains(replicatingPartition)) {
partitionIdIsServedByCurrentNode = true;
break;
}
}
if (partitionIdIsServedByCurrentNode) {
partitions.add(partitionId);
} else {
logger.debug("Partition " + partitionId + " is not served by this node, so it will be skipped.");
}
}
return partitions;
}
use of voldemort.routing.RoutingStrategyFactory in project voldemort by voldemort.
the class AdminRebalanceTest method testRebalanceNodeRW2.
@Test(timeout = 60000)
public void testRebalanceNodeRW2() throws IOException {
try {
startFourNodeRW();
// Start another node for only this unit test
HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(TEST_SIZE);
SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
StoreClient<Object, Object> storeClient1 = factory.getStoreClient("test"), storeClient2 = factory.getStoreClient("test2");
List<Integer> primaryPartitionsMoved = Lists.newArrayList(0);
List<Integer> secondaryPartitionsMoved = Lists.newArrayList(8, 9, 10, 11);
List<Integer> tertiaryPartitionsMoved = Lists.newArrayList(4, 5, 6, 7);
HashMap<ByteArray, byte[]> primaryEntriesMoved = Maps.newHashMap();
HashMap<ByteArray, byte[]> secondaryEntriesMoved = Maps.newHashMap();
HashMap<ByteArray, byte[]> tertiaryEntriesMoved = Maps.newHashMap();
RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef2, currentCluster);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
storeClient1.put(new String(entry.getKey().get()), new String(entry.getValue()));
storeClient2.put(new String(entry.getKey().get()), new String(entry.getValue()));
List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
if (primaryPartitionsMoved.contains(pList.get(0))) {
primaryEntriesMoved.put(entry.getKey(), entry.getValue());
} else if (secondaryPartitionsMoved.contains(pList.get(0))) {
secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
} else if (tertiaryPartitionsMoved.contains(pList.get(0))) {
tertiaryEntriesMoved.put(entry.getKey(), entry.getValue());
}
}
// Set into rebalancing state
for (RebalanceTaskInfo partitionPlan : plans) {
getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
}
// Update the cluster metadata on all three nodes
for (VoldemortServer server : servers) {
server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, finalCluster);
}
// Actually run it
try {
for (RebalanceTaskInfo currentPlan : plans) {
int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
// Test that plan has been removed from the list
assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
}
} catch (Exception e) {
e.printStackTrace();
fail("Should not throw any exceptions");
}
Store<ByteArray, byte[], byte[]> storeTest0 = getStore(0, "test2");
Store<ByteArray, byte[], byte[]> storeTest1 = getStore(1, "test2");
Store<ByteArray, byte[], byte[]> storeTest3 = getStore(3, "test2");
Store<ByteArray, byte[], byte[]> storeTest00 = getStore(0, "test");
Store<ByteArray, byte[], byte[]> storeTest30 = getStore(3, "test");
// Primary
for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
// Test 2
// Present on Node 0
assertSame("entry should be present at store", 1, storeTest0.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest0.get(entry.getKey(), null).get(0).getValue()));
// Present on Node 1
assertSame("entry should be present at store", 1, storeTest1.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest1.get(entry.getKey(), null).get(0).getValue()));
// Present on Node 3
assertSame("entry should be present at store", 1, storeTest3.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest3.get(entry.getKey(), null).get(0).getValue()));
// Test
// Present on Node 0
assertSame("entry should be present at store", 1, storeTest00.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest00.get(entry.getKey(), null).get(0).getValue()));
// Present on Node 3
assertSame("entry should be present at store", 1, storeTest30.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest30.get(entry.getKey(), null).get(0).getValue()));
}
// Secondary
for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
// Test 2
// Present on Node 0
assertSame("entry should be present at store", 1, storeTest0.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest0.get(entry.getKey(), null).get(0).getValue()));
// Present on Node 3
assertSame("entry should be present at store", 1, storeTest3.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest3.get(entry.getKey(), null).get(0).getValue()));
// Test
// Present on Node 3
assertSame("entry should be present at store", 1, storeTest30.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest30.get(entry.getKey(), null).get(0).getValue()));
}
// Tertiary
for (Entry<ByteArray, byte[]> entry : tertiaryEntriesMoved.entrySet()) {
// Test 2
// Present on Node 3
assertSame("entry should be present at store", 1, storeTest3.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest3.get(entry.getKey(), null).get(0).getValue()));
}
// All servers should be back to normal state
for (VoldemortServer server : servers) {
assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
}
} finally {
shutDown();
}
}
use of voldemort.routing.RoutingStrategyFactory in project voldemort by voldemort.
the class AdminRebalanceTest method testRebalanceNodeRW.
@Test(timeout = 60000)
public void testRebalanceNodeRW() throws IOException {
try {
startThreeNodeRW();
// Start another node for only this unit test
HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(TEST_SIZE);
SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
StoreClient<Object, Object> storeClient1 = factory.getStoreClient("test"), storeClient2 = factory.getStoreClient("test2");
List<Integer> primaryPartitionsMoved = Lists.newArrayList(0);
List<Integer> secondaryPartitionsMoved = Lists.newArrayList(4, 5, 6, 7);
HashMap<ByteArray, byte[]> primaryEntriesMoved = Maps.newHashMap();
HashMap<ByteArray, byte[]> secondaryEntriesMoved = Maps.newHashMap();
RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef2, currentCluster);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
storeClient1.put(new String(entry.getKey().get()), new String(entry.getValue()));
storeClient2.put(new String(entry.getKey().get()), new String(entry.getValue()));
List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
if (primaryPartitionsMoved.contains(pList.get(0))) {
primaryEntriesMoved.put(entry.getKey(), entry.getValue());
} else if (secondaryPartitionsMoved.contains(pList.get(0))) {
secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
}
}
try {
adminClient.rebalanceOps.rebalanceNode(plans.get(0));
fail("Should have thrown an exception since not in rebalancing state");
} catch (VoldemortException e) {
}
// Set into rebalancing state
for (RebalanceTaskInfo partitionPlan : plans) {
getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
}
try {
adminClient.rebalanceOps.rebalanceNode(plans.get(0));
fail("Should have thrown an exception since no steal info");
} catch (VoldemortException e) {
}
// Put a plan different from the plan that we actually want to
// execute
int incorrectStealerId = (plans.get(0).getStealerId() + 1) % 3;
getServer(plans.get(0).getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(new RebalanceTaskInfo(incorrectStealerId, plans.get(0).getDonorId(), plans.get(0).getStoreToPartitionIds(), plans.get(0).getInitialCluster()))));
try {
adminClient.rebalanceOps.rebalanceNode(plans.get(0));
fail("Should have thrown an exception since the two plans eventhough have the same donor are different");
} catch (VoldemortException e) {
}
// Set the rebalance info on the stealer node
for (RebalanceTaskInfo partitionPlan : plans) {
getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
}
// Update the cluster metadata on all three nodes
for (VoldemortServer server : servers) {
server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, finalCluster);
}
// Actually run it
try {
for (RebalanceTaskInfo currentPlan : plans) {
int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
// AlreadyRebalancingException
try {
adminClient.rebalanceOps.rebalanceNode(currentPlan);
fail("Should have thrown an exception since it is already rebalancing");
} catch (AlreadyRebalancingException e) {
}
assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
// Test that plan has been removed from the list
assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
}
} catch (Exception e) {
e.printStackTrace();
fail("Should not throw any exceptions");
}
Store<ByteArray, byte[], byte[]> storeTest0 = getStore(0, "test2");
Store<ByteArray, byte[], byte[]> storeTest2 = getStore(2, "test2");
Store<ByteArray, byte[], byte[]> storeTest20 = getStore(2, "test");
// Primary is on Node 0 and not on Node 1
for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
assertSame("entry should be present at store", 1, storeTest0.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest0.get(entry.getKey(), null).get(0).getValue()));
// Check in other store
assertSame("entry should be present in store test2 ", 1, storeTest20.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest20.get(entry.getKey(), null).get(0).getValue()));
}
// Secondary is on Node 2 and not on Node 0
for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
assertSame("entry should be present at store", 1, storeTest2.get(entry.getKey(), null).size());
assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest2.get(entry.getKey(), null).get(0).getValue()));
}
// All servers should be back to normal state
for (VoldemortServer server : servers) {
assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
}
} finally {
shutDown();
}
}
use of voldemort.routing.RoutingStrategyFactory in project voldemort by voldemort.
the class AbstractRebalanceTest method checkGetEntries.
protected void checkGetEntries(Node node, Cluster cluster, StoreDefinition def, Store<ByteArray, byte[], byte[]> store, HashMap<Integer, List<Integer>> flattenedPresentTuples, HashMap<String, String> baselineTuples, HashMap<String, VectorClock> baselineVersions) {
RoutingStrategy routing = new RoutingStrategyFactory().updateRoutingStrategy(def, cluster);
for (Entry<String, String> entry : baselineTuples.entrySet()) {
ByteArray keyBytes = new ByteArray(ByteUtils.getBytes(entry.getKey(), "UTF-8"));
List<Integer> partitions = routing.getPartitionList(keyBytes.get());
if (StoreRoutingPlan.checkKeyBelongsToPartition(partitions, node.getPartitionIds(), flattenedPresentTuples)) {
List<Versioned<byte[]>> values = store.get(keyBytes, null);
// expecting exactly one version
if (values.size() == 0) {
fail("unable to find value for key=" + entry.getKey() + " on node=" + node.getId());
}
assertEquals("Expecting exactly one version", 1, values.size());
Versioned<byte[]> value = values.get(0);
// check version matches
if (baselineVersions == null) {
// expecting base version for all
assertEquals("Value version should match", new VectorClock(), value.getVersion());
} else {
assertEquals("Value version should match", baselineVersions.get(entry.getKey()), value.getVersion());
}
// check value matches.
assertEquals("Value bytes should match", entry.getValue(), ByteUtils.getString(value.getValue(), "UTF-8"));
}
}
}
use of voldemort.routing.RoutingStrategyFactory in project voldemort by voldemort.
the class JsonStoreBuilder method buildVersion2.
public void buildVersion2() throws IOException {
logger.info("Building store " + storeDefinition.getName() + " for " + cluster.getNumberOfPartitions() + " partitions, " + storeDefinition.getReplicationFactor() + " replica types, " + numChunks + " chunks per partitions per replica type and type " + ReadOnlyStorageFormat.READONLY_V2);
// Initialize files
DataOutputStream[][] indexes = new DataOutputStream[cluster.getNumberOfPartitions()][];
DataOutputStream[][] datas = new DataOutputStream[cluster.getNumberOfPartitions()][];
int[][] positions = new int[cluster.getNumberOfPartitions()][];
File tempDirectory = new File(Utils.notNull(System.getProperty("java.io.tmpdir")), "tempDir-" + Integer.toString(new Random().nextInt()));
Utils.mkdirs(tempDirectory);
for (int partitionId = 0; partitionId < cluster.getNumberOfPartitions(); partitionId++) {
indexes[partitionId] = new DataOutputStream[storeDefinition.getReplicationFactor() * numChunks];
datas[partitionId] = new DataOutputStream[storeDefinition.getReplicationFactor() * numChunks];
positions[partitionId] = new int[storeDefinition.getReplicationFactor() * numChunks];
int globalChunkId = 0;
for (int repType = 0; repType < storeDefinition.getReplicationFactor(); repType++) {
for (int chunk = 0; chunk < numChunks; chunk++) {
File indexFile = new File(tempDirectory, Integer.toString(partitionId) + "_" + Integer.toString(repType) + "_" + Integer.toString(chunk) + ".index");
File dataFile = new File(tempDirectory, Integer.toString(partitionId) + "_" + Integer.toString(repType) + "_" + Integer.toString(chunk) + ".data");
positions[partitionId][globalChunkId] = 0;
indexes[partitionId][globalChunkId] = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(indexFile), ioBufferSize));
datas[partitionId][globalChunkId] = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(dataFile), ioBufferSize));
globalChunkId++;
}
}
}
logger.info("Reading items...");
ExternalSorter<KeyValuePair> sorter = new ExternalSorter<KeyValuePair>(new KeyValuePairSerializer(), new KeyMd5Comparator(), internalSortSize, tempDir.getAbsolutePath(), ioBufferSize, numThreads, gzipIntermediate);
JsonObjectIterator iter = new JsonObjectIterator(reader, storeDefinition);
int count = 0;
HashMap<Pair<Integer, Integer>, Pair<byte[], byte[]>> previousElements = Maps.newHashMap();
for (KeyValuePair currentElement : sorter.sorted(iter)) {
List<Integer> partitionIds = this.routingStrategy.getPartitionList(currentElement.getKey());
int masterPartition = partitionIds.get(0);
int localChunkId = ReadOnlyUtils.chunk(currentElement.getKeyMd5(), numChunks);
for (int replicaType = 0; replicaType < partitionIds.size(); replicaType++) {
int globalChunkId = (replicaType * numChunks) + localChunkId;
Pair<Integer, Integer> key = Pair.create(masterPartition, globalChunkId);
if (!previousElements.containsKey(key)) {
// First element, lets write it to map
previousElements.put(key, Pair.create(ByteUtils.copy(currentElement.getKeyMd5(), 0, 2 * ByteUtils.SIZE_OF_INT), generateFirstElement(currentElement)));
} else {
Pair<byte[], byte[]> previousElement = previousElements.get(key);
// append it...
if (ByteUtils.compare(previousElement.getFirst(), currentElement.getKeyMd5(), 0, 2 * ByteUtils.SIZE_OF_INT) == 0) {
short numKeys = ByteUtils.readShort(previousElement.getSecond(), 0);
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream valueStream = new DataOutputStream(stream);
valueStream.writeShort(numKeys + 1);
// Append the previous tuples
valueStream.write(ByteUtils.copy(previousElement.getSecond(), ByteUtils.SIZE_OF_SHORT, previousElement.getSecond().length));
valueStream.writeInt(currentElement.getKey().length);
valueStream.writeInt(currentElement.getValue().length);
valueStream.write(currentElement.getKey());
valueStream.write(currentElement.getValue());
valueStream.flush();
previousElements.put(key, Pair.create(previousElement.getFirst(), stream.toByteArray()));
} else {
// ...else, flush the previous element to disk
indexes[masterPartition][globalChunkId].write(previousElement.getFirst());
indexes[masterPartition][globalChunkId].writeInt(positions[masterPartition][globalChunkId]);
datas[masterPartition][globalChunkId].write(previousElement.getSecond());
positions[masterPartition][globalChunkId] += previousElement.getSecond().length;
// ...and add current element as previous element
previousElements.put(key, Pair.create(ByteUtils.copy(currentElement.getKeyMd5(), 0, 2 * ByteUtils.SIZE_OF_INT), generateFirstElement(currentElement)));
}
}
}
count++;
}
logger.info(count + " items read.");
// files
for (Entry<Pair<Integer, Integer>, Pair<byte[], byte[]>> entry : previousElements.entrySet()) {
int partitionId = entry.getKey().getFirst();
int globalChunkId = entry.getKey().getSecond();
byte[] keyMd5 = entry.getValue().getFirst();
byte[] value = entry.getValue().getSecond();
indexes[partitionId][globalChunkId].write(keyMd5);
indexes[partitionId][globalChunkId].writeInt(positions[partitionId][globalChunkId]);
datas[partitionId][globalChunkId].write(value);
}
// Create node folders
Map<Integer, File> nodeDirs = new HashMap<Integer, File>(cluster.getNumberOfNodes());
for (Node node : cluster.getNodes()) {
int nodeId = node.getId();
// Create data directory
File nodeDir = new File(outputDir, "node-" + Integer.toString(nodeId));
nodeDir.mkdirs();
// Add the data directory to the array
nodeDirs.put(node.getId(), nodeDir);
// Create metadata file
BufferedWriter writer = new BufferedWriter(new FileWriter(new File(nodeDir, ".metadata")));
ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
writer.write(metadata.toJsonString());
writer.close();
}
// Close everything
logger.info("Closing all store files.");
for (int partitionId = 0; partitionId < cluster.getNumberOfPartitions(); partitionId++) {
for (int chunk = 0; chunk < numChunks * storeDefinition.getReplicationFactor(); chunk++) {
indexes[partitionId][chunk].close();
datas[partitionId][chunk].close();
}
}
// Start moving files over to their correct node
RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
Map<Integer, Integer> replicaMapping = cluster.getPartitionIdToNodeIdMap();
for (File file : tempDirectory.listFiles()) {
String fileName = file.getName();
if (fileName.matches("^[\\d]+_[\\d]+_[\\d]+\\.(data|index)")) {
String[] props = fileName.split("_");
int partitionId = Integer.parseInt(props[0]);
int replicaType = Integer.parseInt(props[1]);
int nodeId = replicaMapping.get(strategy.getReplicatingPartitionList(partitionId).get(replicaType));
Utils.move(file, new File(nodeDirs.get(nodeId), fileName));
}
}
}
Aggregations