Examples with Pair voldemort.utils.Pair used on opensource projects

Example 36 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleDeletePartitionEntries.

// TODO : Add ability to use partition scans
public VAdminProto.DeletePartitionEntriesResponse handleDeletePartitionEntries(VAdminProto.DeletePartitionEntriesRequest request) {
    VAdminProto.DeletePartitionEntriesResponse.Builder response = VAdminProto.DeletePartitionEntriesResponse.newBuilder();
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iterator = null;
    try {
        String storeName = request.getStore();
        final List<Integer> partitionsIds = request.getPartitionIdsList();
        final boolean isReadWriteStore = metadataStore.getStoreDef(storeName).getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) != 0;
        if (!isReadWriteStore) {
            throw new VoldemortException("Cannot delete partitions for store " + storeName + " on node " + metadataStore.getNodeId() + " since it is not a RW store");
        }
        StorageEngine<ByteArray, byte[], byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
        VoldemortFilter filter = (request.hasFilter()) ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader) : new DefaultVoldemortFilter();
        EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
        iterator = storageEngine.entries();
        long deleteSuccess = 0;
        logger.info("Deleting entries for RW store " + storeName + " from node " + metadataStore.getNodeId() + " ( " + storeName + " )");
        while (iterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = iterator.next();
            ByteArray key = entry.getFirst();
            Versioned<byte[]> value = entry.getSecond();
            throttler.maybeThrottle(key.length() + valueSize(value));
            if (StoreRoutingPlan.checkKeyBelongsToNode(key.get(), metadataStore.getNodeId(), request.hasInitialCluster() ? new ClusterMapper().readCluster(new StringReader(request.getInitialCluster())) : metadataStore.getCluster(), metadataStore.getStoreDef(storeName)) && filter.accept(key, value)) {
                if (storageEngine.delete(key, value.getVersion())) {
                    deleteSuccess++;
                    if ((deleteSuccess % 10000) == 0) {
                        logger.info(deleteSuccess + " entries deleted from node " + metadataStore.getNodeId() + " for store " + storeName);
                    }
                }
            }
        }
        logger.info("Completed deletion of entries for RW store " + storeName + " from node " + metadataStore.getNodeId() + " ( " + partitionsIds + " )");
        response.setCount(deleteSuccess);
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        logger.error("handleDeletePartitionEntries failed for request(" + request.toString() + ")", e);
    } finally {
        if (null != iterator)
            iterator.close();
    }
    return response.build();
}

Example 37 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class Repartitioner method swapGreedyRandomPartitions.

/**
     * For each node in specified zones, tries swapping some minimum number of
     * random partitions per node with some minimum number of random partitions
     * from other specified nodes. Chooses the best swap in each iteration.
     * Large values of the greedSwapMaxPartitions... arguments make this method
     * equivalent to comparing every possible swap. This may get very expensive.
     * 
     * So if a node had partitions P1, P2, P3 and P4 and the other partitions
     * set was Q1, Q2, Q3, Q4, Q5 The combinations that will be tried for
     * swapping will be the cartesian product of the two sets. That is, {P1,
     * Q1}, {P2, Q2}...{P2,Q1}, {P2,Q2}, in total 20 such swap pairs will be
     * generated. The best among these swap pairs will be chosen.
     * 
     * @param nextCandidateCluster
     * @param nodeIds Node IDs within which to shuffle partitions
     * @param greedySwapMaxPartitionsPerNode See RebalanceCLI.
     * @param greedySwapMaxPartitionsPerZone See RebalanceCLI.
     * @param storeDefs
     * @return updated cluster
     */
public static Cluster swapGreedyRandomPartitions(final Cluster nextCandidateCluster, final List<Integer> nodeIds, final int greedySwapMaxPartitionsPerNode, final int greedySwapMaxPartitionsPerZone, List<StoreDefinition> storeDefs) {
    System.out.println("GreedyRandom : nodeIds:" + nodeIds);
    Cluster returnCluster = Cluster.cloneCluster(nextCandidateCluster);
    double currentUtility = new PartitionBalance(returnCluster, storeDefs).getUtility();
    int nodeIdA = -1;
    int nodeIdB = -1;
    int partitionIdA = -1;
    int partitionIdB = -1;
    for (int nodeIdAPrime : nodeIds) {
        System.out.println("GreedyRandom : processing nodeId:" + nodeIdAPrime);
        List<Integer> partitionIdsAPrime = new ArrayList<Integer>();
        partitionIdsAPrime.addAll(returnCluster.getNodeById(nodeIdAPrime).getPartitionIds());
        Collections.shuffle(partitionIdsAPrime);
        int maxPartitionsInAPrime = Math.min(greedySwapMaxPartitionsPerNode, partitionIdsAPrime.size());
        for (int offsetAPrime = 0; offsetAPrime < maxPartitionsInAPrime; offsetAPrime++) {
            Integer partitionIdAPrime = partitionIdsAPrime.get(offsetAPrime);
            List<Pair<Integer, Integer>> partitionIdsZone = new ArrayList<Pair<Integer, Integer>>();
            for (int nodeIdBPrime : nodeIds) {
                if (nodeIdBPrime == nodeIdAPrime)
                    continue;
                for (Integer partitionIdBPrime : returnCluster.getNodeById(nodeIdBPrime).getPartitionIds()) {
                    partitionIdsZone.add(new Pair<Integer, Integer>(nodeIdBPrime, partitionIdBPrime));
                }
            }
            Collections.shuffle(partitionIdsZone);
            int maxPartitionsInZone = Math.min(greedySwapMaxPartitionsPerZone, partitionIdsZone.size());
            for (int offsetZone = 0; offsetZone < maxPartitionsInZone; offsetZone++) {
                Integer nodeIdBPrime = partitionIdsZone.get(offsetZone).getFirst();
                Integer partitionIdBPrime = partitionIdsZone.get(offsetZone).getSecond();
                Cluster swapResult = swapPartitions(returnCluster, nodeIdAPrime, partitionIdAPrime, nodeIdBPrime, partitionIdBPrime);
                double swapUtility = new PartitionBalance(swapResult, storeDefs).getUtility();
                if (swapUtility < currentUtility) {
                    currentUtility = swapUtility;
                    System.out.println(" -> " + currentUtility);
                    nodeIdA = nodeIdAPrime;
                    partitionIdA = partitionIdAPrime;
                    nodeIdB = nodeIdBPrime;
                    partitionIdB = partitionIdBPrime;
                }
            }
        }
    }
    if (nodeIdA == -1) {
        return returnCluster;
    }
    return swapPartitions(returnCluster, nodeIdA, partitionIdA, nodeIdB, partitionIdB);
}

Example 38 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class Repartitioner method getDonorsAndStealersForBalance.

/**
     * Assign target number of partitions per node to specific node IDs. Then,
     * separates Nodes into donorNodes and stealerNodes based on whether the
     * node needs to donate or steal primary partitions.
     * 
     * @param nextCandidateCluster
     * @param numPartitionsPerNodePerZone
     * @return a Pair. First element is donorNodes, second element is
     *         stealerNodes. Each element in the pair is a HashMap of Node to
     *         Integer where the integer value is the number of partitions to
     *         store.
     */
public static Pair<HashMap<Node, Integer>, HashMap<Node, Integer>> getDonorsAndStealersForBalance(final Cluster nextCandidateCluster, Map<Integer, List<Integer>> numPartitionsPerNodePerZone) {
    HashMap<Node, Integer> donorNodes = Maps.newHashMap();
    HashMap<Node, Integer> stealerNodes = Maps.newHashMap();
    HashMap<Integer, Integer> numNodesAssignedInZone = Maps.newHashMap();
    for (Integer zoneId : nextCandidateCluster.getZoneIds()) {
        numNodesAssignedInZone.put(zoneId, 0);
    }
    for (Node node : nextCandidateCluster.getNodes()) {
        int zoneId = node.getZoneId();
        int offset = numNodesAssignedInZone.get(zoneId);
        numNodesAssignedInZone.put(zoneId, offset + 1);
        int numPartitions = numPartitionsPerNodePerZone.get(zoneId).get(offset);
        if (numPartitions < node.getNumberOfPartitions()) {
            donorNodes.put(node, numPartitions);
        } else if (numPartitions > node.getNumberOfPartitions()) {
            stealerNodes.put(node, numPartitions);
        }
    }
    // Print out donor/stealer information
    for (Node node : donorNodes.keySet()) {
        System.out.println("Donor Node: " + node.getId() + ", zoneId " + node.getZoneId() + ", numPartitions " + node.getNumberOfPartitions() + ", target number of partitions " + donorNodes.get(node));
    }
    for (Node node : stealerNodes.keySet()) {
        System.out.println("Stealer Node: " + node.getId() + ", zoneId " + node.getZoneId() + ", numPartitions " + node.getNumberOfPartitions() + ", target number of partitions " + stealerNodes.get(node));
    }
    return new Pair<HashMap<Node, Integer>, HashMap<Node, Integer>>(donorNodes, stealerNodes);
}

Example 39 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class ImportTextDumpToBDB method lineToEntry.

public static Pair<ByteArray, Versioned<byte[]>> lineToEntry(String line) throws DecoderException {
    String[] components = line.split(" ");
    String keyBytesString = components[0];
    byte[] keyBytes = ByteUtils.fromHexString(keyBytesString);
    ByteArray key = new ByteArray(keyBytes);
    String versionBytesString = components[1];
    byte[] versionBytes = ByteUtils.fromHexString(versionBytesString);
    Version version = new VectorClock(versionBytes, 0);
    String valueBytesString = components[1];
    byte[] value = ByteUtils.fromHexString(valueBytesString);
    return new Pair<ByteArray, Versioned<byte[]>>(key, new Versioned<byte[]>(value, version));
}

Example 40 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class DataCleanupJob method run.

@Override
public void run() {
    // if the server is neither normal nor offline , skip this run.
    if (!isServerInNormalState() && !isServerInOfflineState()) {
        return;
    }
    StoreDefinition storeDef = null;
    try {
        storeDef = MetadataStore.getStoreDef(storeName, metadataStore);
    } catch (VoldemortException ex) {
        logger.info("Error retrieving store " + storeName + " for data cleanup job ", ex);
        return;
    }
    Integer retentionDays = storeDef.getRetentionDays();
    if (retentionDays == null || retentionDays <= 0) {
        logger.info("Store " + storeName + " does not have retention period set, skipping cleanup job . RetentionDays " + retentionDays);
        return;
    }
    long maxAgeMs = retentionDays * Time.MS_PER_DAY;
    logger.info("Store " + storeName + " cleanup job is starting with RetentionDays " + retentionDays);
    acquireCleanupPermit(scanProgressThisRun, deleteProgressThisRun);
    ClosableIterator<Pair<K, Versioned<V>>> iterator = null;
    try {
        int maxReadRate = storeDef.hasRetentionScanThrottleRate() ? storeDef.getRetentionScanThrottleRate() : Integer.MAX_VALUE;
        EventThrottler throttler = new EventThrottler(maxReadRate);
        store.beginBatchModifications();
        logger.info("Starting data cleanup on store \"" + store.getName() + "\"...");
        long now = time.getMilliseconds();
        iterator = store.entries();
        while (iterator.hasNext()) {
            // check if we have been interrupted
            if (Thread.currentThread().isInterrupted()) {
                logger.info("Datacleanup job halted.");
                return;
            }
            final long INETERVAL = 10000;
            long entriesScanned = scanProgressThisRun.get();
            if (entriesScanned % INETERVAL == 0) {
                if (!isServerInNormalState() && !isServerInOfflineState()) {
                    return;
                }
            }
            scanProgressThisRun.incrementAndGet();
            Pair<K, Versioned<V>> keyAndVal = iterator.next();
            VectorClock clock = (VectorClock) keyAndVal.getSecond().getVersion();
            if (now - clock.getTimestamp() > maxAgeMs) {
                store.delete(keyAndVal.getFirst(), clock);
                final long entriesDeleted = this.deleteProgressThisRun.incrementAndGet();
                if (logger.isDebugEnabled() && entriesDeleted % INETERVAL == 0) {
                    logger.debug("Deleted item " + this.deleteProgressThisRun.get());
                }
            }
            // throttle on number of entries.
            throttler.maybeThrottle(1);
        }
        // log the total items scanned, so we will get an idea of data
        // growth in a cheap, periodic way
        logger.info("Data cleanup on store \"" + store.getName() + "\" is complete; " + this.deleteProgressThisRun.get() + " items deleted. " + scanProgressThisRun.get() + " items scanned");
    } catch (Exception e) {
        logger.error("Error in data cleanup job for store " + store.getName() + ": ", e);
    } finally {
        closeIterator(iterator);
        logger.info("Releasing lock  after data cleanup on \"" + store.getName() + "\".");
        this.cleanupPermits.release(this.getClass().getCanonicalName());
        synchronized (this) {
            totalEntriesScanned += scanProgressThisRun.get();
            scanProgressThisRun.set(0);
            totalEntriesDeleted += deleteProgressThisRun.get();
            deleteProgressThisRun.set(0);
        }
        store.endBatchModifications();
    }
}