Examples with RebalanceTaskInfo voldemort.client.rebalance.RebalanceTaskInfo used on opensource projects

Example 6 with RebalanceTaskInfo

use of voldemort.client.rebalance.RebalanceTaskInfo in project voldemort by voldemort.

the class Rebalancer method rebalanceNode.

/**
     * This function is responsible for starting the actual async rebalance
     * operation. This is run if this node is the stealer node
     * 
     * <br>
     * 
     * We also assume that the check that this server is in rebalancing state
     * has been done at a higher level
     * 
     * @param stealInfo Partition info to steal
     * @return Returns a id identifying the async operation
     */
public int rebalanceNode(final RebalanceTaskInfo stealInfo) {
    final RebalanceTaskInfo info = metadataStore.getRebalancerState().find(stealInfo.getDonorId());
    // Do we have the plan in the state?
    if (info == null) {
        throw new VoldemortException("Could not find plan " + stealInfo + " in the server state on " + metadataStore.getNodeId());
    } else if (!info.equals(stealInfo)) {
        // If we do have the plan, is it the same
        throw new VoldemortException("The plan in server state " + info + " is not the same as the process passed " + stealInfo);
    } else if (!acquireRebalancingPermit(stealInfo.getDonorId())) {
        // Both are same, now try to acquire a lock for the donor node
        throw new AlreadyRebalancingException("Node " + metadataStore.getNodeId() + " is already rebalancing from donor " + info.getDonorId() + " with info " + info);
    }
    // Acquired lock successfully, start rebalancing...
    int requestId = asyncService.getUniqueRequestId();
    // Why do we pass 'info' instead of 'stealInfo'? So that we can change
    // the state as the stores finish rebalance
    asyncService.submitOperation(requestId, new StealerBasedRebalanceAsyncOperation(this, voldemortConfig, metadataStore, requestId, info));
    return requestId;
}

Example 7 with RebalanceTaskInfo

use of voldemort.client.rebalance.RebalanceTaskInfo in project voldemort by voldemort.

the class RebalanceUtils method groupPartitionsTaskByNode.

/**
     * Given a list of partition infos, generates a map of stealer / donor node
     * to list of partition infos
     * 
     * @param rebalanceTaskPlanList Complete list of partition plans
     * @param groupByStealerNode Boolean indicating if we want to group by
     *        stealer node ( or donor node )
     * @return Flattens it into a map on a per node basis
     */
public static HashMap<Integer, List<RebalanceTaskInfo>> groupPartitionsTaskByNode(List<RebalanceTaskInfo> rebalanceTaskPlanList, boolean groupByStealerNode) {
    HashMap<Integer, List<RebalanceTaskInfo>> nodeToTaskInfo = Maps.newHashMap();
    if (rebalanceTaskPlanList != null) {
        for (RebalanceTaskInfo partitionInfo : rebalanceTaskPlanList) {
            int nodeId = groupByStealerNode ? partitionInfo.getStealerId() : partitionInfo.getDonorId();
            List<RebalanceTaskInfo> taskInfos = nodeToTaskInfo.get(nodeId);
            if (taskInfos == null) {
                taskInfos = Lists.newArrayList();
                nodeToTaskInfo.put(nodeId, taskInfos);
            }
            taskInfos.add(partitionInfo);
        }
    }
    return nodeToTaskInfo;
}

Example 8 with RebalanceTaskInfo

use of voldemort.client.rebalance.RebalanceTaskInfo in project voldemort by voldemort.

the class RedirectingStoreTest method setUp.

@Before
public void setUp() throws IOException, InterruptedException {
    currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1 }, { 2, 3 }, {} });
    targetCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Arrays.asList(0));
    this.primaryPartitionsMoved = Lists.newArrayList(0);
    this.secondaryPartitionsMoved = Lists.newArrayList(2, 3);
    this.storeDef = new StoreDefinitionBuilder().setName("test").setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    File tempStoreXml = new File(TestUtils.createTempDir(), "stores.xml");
    FileUtils.writeStringToFile(tempStoreXml, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(storeDef)));
    this.servers = new VoldemortServer[3];
    for (int nodeId = 0; nodeId < 3; nodeId++) {
        this.servers[nodeId] = startServer(nodeId, tempStoreXml.getAbsolutePath(), currentCluster);
    }
    // Start another node for only this unit test
    HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(100);
    SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
    StoreClient<Object, Object> storeClient = factory.getStoreClient("test");
    this.primaryEntriesMoved = Maps.newHashMap();
    this.secondaryEntriesMoved = Maps.newHashMap();
    this.proxyPutTestPrimaryEntries = Maps.newHashMap();
    this.proxyPutTestSecondaryEntries = Maps.newHashMap();
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, currentCluster);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
        storeClient.put(new String(entry.getKey().get()), new String(entry.getValue()));
        List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
        if (primaryPartitionsMoved.contains(pList.get(0))) {
            primaryEntriesMoved.put(entry.getKey(), entry.getValue());
        } else if (secondaryPartitionsMoved.contains(pList.get(0))) {
            secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
        }
    }
    // Sleep a while for the queries to go through...
    // Hope the 'God of perfect timing' is on our side
    Thread.sleep(500);
    // steal a few primary key-value pairs for testing proxy put logic
    int cnt = 0;
    for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestPrimaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestPrimaryEntries.keySet()) {
        this.primaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough primary entries", primaryEntriesMoved.size() > 1);
    // steal a few secondary key-value pairs for testing proxy put logic
    cnt = 0;
    for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestSecondaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestSecondaryEntries.keySet()) {
        this.secondaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough secondary entries", primaryEntriesMoved.size() > 1);
    RebalanceBatchPlan RebalanceBatchPlan = new RebalanceBatchPlan(currentCluster, targetCluster, Lists.newArrayList(storeDef));
    List<RebalanceTaskInfo> plans = Lists.newArrayList(RebalanceBatchPlan.getBatchPlan());
    // Set into rebalancing state
    for (RebalanceTaskInfo partitionPlan : plans) {
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(partitionPlan)));
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, currentCluster);
        // update original storedefs
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_STORES_XML, Lists.newArrayList(storeDef));
    }
    // Update the cluster metadata on all three nodes
    for (VoldemortServer server : servers) {
        server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, targetCluster);
    }
}

Example 9 with RebalanceTaskInfo

use of voldemort.client.rebalance.RebalanceTaskInfo in project voldemort by voldemort.

the class RebalancerStateTest method testToJson.

@Test
public void testToJson() {
    HashMap<String, List<Integer>> storeToPartitionIds = Maps.newHashMap();
    storeToPartitionIds.put("test1", Lists.newArrayList(0, 1, 2));
    storeToPartitionIds.put("test2", Lists.newArrayList(3, 4, 5, 6));
    List<RebalanceTaskInfo> rebalanceTaskInfos = Arrays.asList(new RebalanceTaskInfo(2, 0, storeToPartitionIds, ServerTestUtils.getLocalCluster(1)), new RebalanceTaskInfo(3, 1, storeToPartitionIds, ServerTestUtils.getLocalCluster(2)));
    RebalancerState in = new RebalancerState(rebalanceTaskInfos);
    String jsonIn = in.toJsonString();
    RebalancerState out = RebalancerState.create(jsonIn);
    assertEquals(in, out);
    String jsonOut = out.toJsonString();
    assertEquals(jsonIn, jsonOut);
}

Example 10 with RebalanceTaskInfo

use of voldemort.client.rebalance.RebalanceTaskInfo in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleRebalanceNode.

public VAdminProto.AsyncOperationStatusResponse handleRebalanceNode(VAdminProto.InitiateRebalanceNodeRequest request) {
    VAdminProto.AsyncOperationStatusResponse.Builder response = VAdminProto.AsyncOperationStatusResponse.newBuilder();
    try {
        if (!voldemortConfig.isEnableRebalanceService())
            throw new VoldemortException("Rebalance service is not enabled for node: " + metadataStore.getNodeId());
        // We should be in rebalancing state to run this function
        if (!metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
            response.setError(ProtoUtils.encodeError(errorCodeMapper, new VoldemortException("Voldemort server " + metadataStore.getNodeId() + " not in rebalancing state")));
            return response.build();
        }
        RebalanceTaskInfo rebalanceStealInfo = ProtoUtils.decodeRebalanceTaskInfoMap(request.getRebalanceTaskInfo());
        int requestId = rebalancer.rebalanceNode(rebalanceStealInfo);
        response.setRequestId(requestId).setDescription(rebalanceStealInfo.toString()).setStatus("Started rebalancing").setComplete(false);
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleRebalanceNode failed for request(" + request.toString() + ")", e);
    }
    return response.build();
}