Examples with Pair voldemort.utils.Pair used on opensource projects

Example 31 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class StreamingSlopPusherJob method run.

public void run() {
    // load the metadata before each run, in case the cluster is changed
    loadMetadata();
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    boolean terminatedEarly = false;
    Date startTime = new Date();
    logger.info("Started streaming slop pusher job at " + startTime);
    SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    if (adminClient == null) {
        adminClient = new AdminClient(cluster, new AdminClientConfig().setMaxConnectionsPerNode(1));
    }
    if (voldemortConfig.getSlopZonesDownToTerminate() > 0) {
        // Populating the zone mapping for early termination
        zoneMapping.clear();
        for (Node n : cluster.getNodes()) {
            if (failureDetector.isAvailable(n)) {
                Set<Integer> nodes = zoneMapping.get(n.getZoneId());
                if (nodes == null) {
                    nodes = Sets.newHashSet();
                    zoneMapping.put(n.getZoneId(), nodes);
                }
                nodes.add(n.getId());
            }
        }
        // Check how many zones are down
        int zonesDown = 0;
        for (Zone zone : cluster.getZones()) {
            if (zoneMapping.get(zone.getId()) == null || zoneMapping.get(zone.getId()).size() == 0)
                zonesDown++;
        }
        // Terminate early
        if (voldemortConfig.getSlopZonesDownToTerminate() <= zoneMapping.size() && zonesDown >= voldemortConfig.getSlopZonesDownToTerminate()) {
            logger.info("Completed streaming slop pusher job at " + startTime + " early because " + zonesDown + " zones are down");
            stopAdminClient();
            return;
        }
    }
    // Clearing the statistics
    AtomicLong attemptedPushes = new AtomicLong(0);
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    acquireRepairPermit();
    try {
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            Pair<ByteArray, Versioned<Slop>> keyAndVal;
            try {
                keyAndVal = iterator.next();
                Versioned<Slop> versioned = keyAndVal.getSecond();
                // Track the scan progress
                if (this.streamStats != null) {
                    this.streamStats.reportStreamingSlopScan();
                }
                // Retrieve the node
                int nodeId = versioned.getValue().getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // ignore it.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes.incrementAndGet();
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (attemptedPushes.get() % 10000 == 0)
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                if (logger.isTraceEnabled())
                    logger.trace("Pushing slop for " + versioned.getValue().getNodeId() + " and store  " + versioned.getValue().getStoreName() + " of key: " + versioned.getValue().getKey());
                if (failureDetector.isAvailable(node)) {
                    SynchronousQueue<Versioned<Slop>> slopQueue = slopQueues.get(nodeId);
                    if (slopQueue == null) {
                        // No previous slop queue, add one
                        slopQueue = new SynchronousQueue<Versioned<Slop>>();
                        slopQueues.put(nodeId, slopQueue);
                        consumerResults.add(consumerExecutor.submit(new SlopConsumer(nodeId, slopQueue, slopStorageEngine)));
                    }
                    boolean offered = slopQueue.offer(versioned, voldemortConfig.getClientRoutingTimeoutMs(), TimeUnit.MILLISECONDS);
                    if (!offered) {
                        if (logger.isDebugEnabled())
                            logger.debug("No consumer appeared for slop in " + voldemortConfig.getClientConnectionTimeoutMs() + " ms");
                    }
                    readThrottler.maybeThrottle(nBytesRead(keyAndVal));
                } else {
                    logger.trace(node + " declared down, won't push slop");
                }
            } catch (RejectedExecutionException e) {
                throw new VoldemortException("Ran out of threads in executor", e);
            }
        }
    } catch (InterruptedException e) {
        logger.warn("Interrupted exception", e);
        terminatedEarly = true;
    } catch (Exception e) {
        logger.error(e, e);
        terminatedEarly = true;
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.warn("Failed to close iterator cleanly as database might be closed", e);
        }
        // Adding the poison pill
        for (SynchronousQueue<Versioned<Slop>> slopQueue : slopQueues.values()) {
            try {
                slopQueue.put(END);
            } catch (InterruptedException e) {
                logger.warn("Error putting poison pill", e);
            }
        }
        for (Future result : consumerResults) {
            try {
                result.get();
            } catch (Exception e) {
                logger.warn("Exception in consumer", e);
            }
        }
        // Only if exception didn't take place do we update the counts
        if (!terminatedEarly) {
            Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
            for (int nodeId : succeededByNode.keySet()) {
                logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
                outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
            }
            slopStorageEngine.resetStats(outstanding);
            logger.info("Completed streaming slop pusher job which started at " + startTime);
        } else {
            for (int nodeId : succeededByNode.keySet()) {
                logger.info("Slops to node " + nodeId + " - Succeeded - " + succeededByNode.get(nodeId) + " - Attempted - " + attemptedByNode.get(nodeId));
            }
            logger.info("Completed early streaming slop pusher job which started at " + startTime);
        }
        // Shut down admin client as not to waste connections
        consumerResults.clear();
        slopQueues.clear();
        stopAdminClient();
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}

Example 32 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class MetadataStore method initStoreDefinitions.

/**
     * Function to go through all the store definitions contained in the STORES
     * directory and
     * 
     * 1. Update metadata cache.
     * 
     * 2. Update STORES_KEY by stitching together all these keys.
     * 
     * 3. Update 'storeNames' list.
     * 
     * This method is not thread safe. It is expected that the caller of this
     * method will correctly handle concurrency issues. Currently this is not an
     * issue since its invoked by init, put, add and delete store all of which
     * use locks to deal with any concurrency related issues.
     */
private void initStoreDefinitions(Version storesXmlVersion) {
    if (this.storeDefinitionsStorageEngine == null) {
        throw new VoldemortException("The store definitions directory is empty");
    }
    String allStoreDefinitions = "<stores>";
    Version finalStoresXmlVersion = null;
    if (storesXmlVersion != null) {
        finalStoresXmlVersion = storesXmlVersion;
    }
    this.storeNames.clear();
    ClosableIterator<Pair<String, Versioned<String>>> storesIterator = this.storeDefinitionsStorageEngine.entries();
    // Some test setups may result in duplicate entries for 'store' element.
    // Do the de-dup here
    Map<String, Versioned<String>> storeNameToDefMap = new HashMap<String, Versioned<String>>();
    Version maxVersion = null;
    while (storesIterator.hasNext()) {
        Pair<String, Versioned<String>> storeDetail = storesIterator.next();
        String storeName = storeDetail.getFirst();
        Versioned<String> versionedStoreDef = storeDetail.getSecond();
        storeNameToDefMap.put(storeName, versionedStoreDef);
        Version curVersion = versionedStoreDef.getVersion();
        // Get the highest version from all the store entries
        if (maxVersion == null) {
            maxVersion = curVersion;
        } else if (maxVersion.compare(curVersion) == Occurred.BEFORE) {
            maxVersion = curVersion;
        }
    }
    // 'stores.xml' key
    if (finalStoresXmlVersion == null) {
        finalStoresXmlVersion = maxVersion;
    }
    // Go through all the individual stores and update metadata
    for (Entry<String, Versioned<String>> storeEntry : storeNameToDefMap.entrySet()) {
        String storeName = storeEntry.getKey();
        Versioned<String> versionedStoreDef = storeEntry.getValue();
        // Add all the store names to the list of storeNames
        this.storeNames.add(storeName);
        this.metadataCache.put(storeName, new Versioned<Object>(versionedStoreDef.getValue(), versionedStoreDef.getVersion()));
    }
    Collections.sort(this.storeNames);
    for (String storeName : this.storeNames) {
        Versioned<String> versionedStoreDef = storeNameToDefMap.get(storeName);
        // Stitch together to form the complete store definition list.
        allStoreDefinitions += versionedStoreDef.getValue();
    }
    allStoreDefinitions += "</stores>";
    // Update cache with the composite store definition list.
    metadataCache.put(STORES_KEY, convertStringToObject(STORES_KEY, new Versioned<String>(allStoreDefinitions, finalStoresXmlVersion)));
}

Example 33 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class AdminRebalanceTest method checkRO.

private void checkRO(Cluster cluster) {
    for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
        Map<Integer, Set<Pair<Integer, Integer>>> nodeToPartitions = ROTestUtils.getNodeIdToAllPartitions(cluster, storeDef, true);
        for (Map.Entry<Integer, Set<Pair<Integer, Integer>>> entry : nodeToPartitions.entrySet()) {
            int nodeId = entry.getKey();
            Set<Pair<Integer, Integer>> buckets = entry.getValue();
            assertEquals(servers[nodeId].getMetadataStore().getCluster(), cluster);
            ReadOnlyStorageEngine engine = (ReadOnlyStorageEngine) servers[nodeId].getStoreRepository().getStorageEngine(storeDef.getName());
            HashMap<Object, Integer> storeBuckets = engine.getChunkedFileSet().getChunkIdToNumChunks();
            for (Pair<Integer, Integer> bucket : buckets) {
                if (bucket.getFirst() < storeDef.getReplicationFactor())
                    assertEquals(storeBuckets.containsKey(Pair.create(bucket.getSecond(), bucket.getFirst())), true);
            }
        }
    }
}

Example 34 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class BaseStreamingClient method commitToVoldemort.

/**
     * Flush the network buffer and write all entries to the serve. then wait
     * for an ack from the server. This is a blocking call. It is invoked on
     * every Commit batch size of entries, It is also called on the close
     * session call
     * 
     * @param storeNamesToCommit List of stores to be flushed and committed
     * 
     */
@SuppressWarnings({ "unchecked", "rawtypes", "unused" })
private void commitToVoldemort(List<String> storeNamesToCommit) {
    if (logger.isDebugEnabled()) {
        logger.debug("Trying to commit to Voldemort");
    }
    boolean hasError = false;
    if (nodesToStream == null || nodesToStream.size() == 0) {
        if (logger.isDebugEnabled()) {
            logger.debug("No nodes to stream to. Returning.");
        }
        return;
    }
    for (Node node : nodesToStream) {
        for (String store : storeNamesToCommit) {
            if (!nodeIdStoreInitialized.get(new Pair(store, node.getId())))
                continue;
            nodeIdStoreInitialized.put(new Pair(store, node.getId()), false);
            DataOutputStream outputStream = nodeIdStoreToOutputStreamRequest.get(new Pair(store, node.getId()));
            try {
                ProtoUtils.writeEndOfStream(outputStream);
                outputStream.flush();
                DataInputStream inputStream = nodeIdStoreToInputStreamRequest.get(new Pair(store, node.getId()));
                VAdminProto.UpdatePartitionEntriesResponse.Builder updateResponse = ProtoUtils.readToBuilder(inputStream, VAdminProto.UpdatePartitionEntriesResponse.newBuilder());
                if (updateResponse.hasError()) {
                    hasError = true;
                }
            } catch (IOException e) {
                logger.error("Exception during commit", e);
                hasError = true;
                if (!faultyNodes.contains(node.getId()))
                    faultyNodes.add(node.getId());
            }
        }
    }
    if (streamingresults == null) {
        logger.warn("StreamingSession may not have been initialized since Variable streamingresults is null. Skipping callback ");
        return;
    }
    // remove redundant callbacks
    if (hasError) {
        logger.info("Invoking the Recovery Callback");
        Future future = streamingresults.submit(recoveryCallback);
        try {
            future.get();
        } catch (InterruptedException e1) {
            MARKED_BAD = true;
            logger.error("Recovery Callback failed", e1);
            throw new VoldemortException("Recovery Callback failed");
        } catch (ExecutionException e1) {
            MARKED_BAD = true;
            logger.error("Recovery Callback failed during execution", e1);
            throw new VoldemortException("Recovery Callback failed during execution");
        }
    } else {
        if (logger.isDebugEnabled()) {
            logger.debug("Commit successful");
            logger.debug("calling checkpoint callback");
        }
        Future future = streamingresults.submit(checkpointCallback);
        try {
            future.get();
        } catch (InterruptedException e1) {
            logger.warn("Checkpoint callback failed!", e1);
        } catch (ExecutionException e1) {
            logger.warn("Checkpoint callback failed during execution!", e1);
        }
    }
}

Example 35 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class BaseStreamingClient method cleanupSessions.

/**
     * Helper method to Close all open socket connections and checkin back to
     * the pool
     * 
     * @param storeNamesToCleanUp List of stores to be cleanedup from the current
     *        streaming session
     */
@SuppressWarnings({ "rawtypes", "unchecked" })
private void cleanupSessions(List<String> storeNamesToCleanUp) {
    logger.info("Performing cleanup");
    for (String store : storeNamesToCleanUp) {
        for (Node node : nodesToStream) {
            try {
                SocketAndStreams sands = nodeIdStoreToSocketAndStreams.get(new Pair(store, node.getId()));
                close(sands.getSocket());
                SocketDestination destination = nodeIdStoreToSocketRequest.get(new Pair(store, node.getId()));
                streamingSocketPool.checkin(destination, sands);
            } catch (Exception ioE) {
                logger.error(ioE);
            }
        }
    }
    cleanedUp = true;
}