Examples with Pair voldemort.utils.Pair used on opensource projects

Example 16 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class BdbStorageEngineTest method testSimultaneousIterationAndModification.

@Test
public void testSimultaneousIterationAndModification() throws Exception {
    // start a thread to do modifications
    ExecutorService executor = Executors.newFixedThreadPool(2);
    final Random rand = new Random();
    final AtomicInteger count = new AtomicInteger(0);
    final AtomicBoolean keepRunning = new AtomicBoolean(true);
    executor.execute(new Runnable() {

        public void run() {
            while (keepRunning.get()) {
                byte[] bytes = Integer.toString(count.getAndIncrement()).getBytes();
                store.put(new ByteArray(bytes), Versioned.value(bytes), null);
                count.incrementAndGet();
            }
        }
    });
    executor.execute(new Runnable() {

        public void run() {
            while (keepRunning.get()) {
                byte[] bytes = Integer.toString(rand.nextInt(count.get())).getBytes();
                store.delete(new ByteArray(bytes), new VectorClock());
                count.incrementAndGet();
            }
        }
    });
    // wait a bit
    while (count.get() < 300) continue;
    // now simultaneously do iteration
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iter = this.store.entries();
    while (iter.hasNext()) iter.next();
    iter.close();
    keepRunning.set(false);
    executor.shutdown();
    assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
}

Example 17 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class BaseStreamingClient method addStoreToSession.

/**
     * Add another store destination to an existing streaming session
     * 
     * 
     * @param store the name of the store to stream to
     */
@SuppressWarnings({ "unchecked", "rawtypes" })
protected void addStoreToSession(String store) {
    Exception initializationException = null;
    storeNames.add(store);
    for (Node node : nodesToStream) {
        SocketDestination destination = null;
        SocketAndStreams sands = null;
        try {
            destination = new SocketDestination(node.getHost(), node.getAdminPort(), RequestFormatType.ADMIN_PROTOCOL_BUFFERS);
            sands = streamingSocketPool.checkout(destination);
            DataOutputStream outputStream = sands.getOutputStream();
            DataInputStream inputStream = sands.getInputStream();
            nodeIdStoreToSocketRequest.put(new Pair(store, node.getId()), destination);
            nodeIdStoreToOutputStreamRequest.put(new Pair(store, node.getId()), outputStream);
            nodeIdStoreToInputStreamRequest.put(new Pair(store, node.getId()), inputStream);
            nodeIdStoreToSocketAndStreams.put(new Pair(store, node.getId()), sands);
            nodeIdStoreInitialized.put(new Pair(store, node.getId()), false);
            remoteStoreDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(node.getId()).getValue();
        } catch (Exception e) {
            logger.error(e);
            try {
                close(sands.getSocket());
                streamingSocketPool.checkin(destination, sands);
            } catch (Exception ioE) {
                logger.error(ioE);
            }
            if (!faultyNodes.contains(node.getId()))
                faultyNodes.add(node.getId());
            initializationException = e;
        }
    }
    if (initializationException != null)
        throw new VoldemortException(initializationException);
    if (store.equals("slop"))
        return;
    boolean foundStore = false;
    for (StoreDefinition remoteStoreDef : remoteStoreDefs) {
        if (remoteStoreDef.getName().equals(store)) {
            RoutingStrategyFactory factory = new RoutingStrategyFactory();
            RoutingStrategy storeRoutingStrategy = factory.updateRoutingStrategy(remoteStoreDef, adminClient.getAdminClientCluster());
            storeToRoutingStrategy.put(store, storeRoutingStrategy);
            validateSufficientNodesAvailable(blackListedNodes, remoteStoreDef);
            foundStore = true;
            break;
        }
    }
    if (!foundStore) {
        logger.error("Store Name not found on the cluster");
        throw new VoldemortException("Store Name not found on the cluster");
    }
}

Example 18 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class BaseStreamingClient method blacklistNode.

/**
     * mark a node as blacklisted
     * 
     * @param nodeId Integer node id of the node to be blacklisted
     */
@SuppressWarnings({ "rawtypes", "unchecked" })
public void blacklistNode(int nodeId) {
    Collection<Node> nodesInCluster = adminClient.getAdminClientCluster().getNodes();
    if (blackListedNodes == null) {
        blackListedNodes = new ArrayList();
    }
    blackListedNodes.add(nodeId);
    for (Node node : nodesInCluster) {
        if (node.getId() == nodeId) {
            nodesToStream.remove(node);
            break;
        }
    }
    for (String store : storeNames) {
        try {
            SocketAndStreams sands = nodeIdStoreToSocketAndStreams.get(new Pair(store, nodeId));
            close(sands.getSocket());
            SocketDestination destination = nodeIdStoreToSocketRequest.get(new Pair(store, nodeId));
            streamingSocketPool.checkin(destination, sands);
        } catch (Exception ioE) {
            logger.error(ioE);
        }
    }
}

Example 19 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class BaseStreamingClient method streamingPut.

/**
     ** 
     * @param key - The key
     * 
     * @param value - The value
     * 
     * @param storeName takes an additional store name as a parameter
     * 
     *        If a store is added mid way through a streaming session we do not
     *        play catchup and entries that were processed earlier during the
     *        session will not be applied for the store.
     * 
     **/
@SuppressWarnings({ "unchecked", "rawtypes" })
public synchronized void streamingPut(ByteArray key, Versioned<byte[]> value, String storeName) {
    // add it and checkout a socket
    if (!storeNames.contains(storeName)) {
        addStoreToSession(storeName);
    }
    if (MARKED_BAD) {
        logger.error("Cannot stream more entries since Recovery Callback Failed!");
        throw new VoldemortException("Cannot stream more entries since Recovery Callback Failed! You Need to restart the session");
    }
    List<Node> nodeList = storeToRoutingStrategy.get(storeName).routeRequest(key.get());
    int nodesWithException = 0;
    // sent the k/v pair to the nodes
    for (Node node : nodeList) {
        if (blackListedNodes != null && blackListedNodes.size() > 0) {
            if (blackListedNodes.contains(node.getId()))
                continue;
        }
        // if node! in blacklistednodes
        VAdminProto.PartitionEntry partitionEntry = VAdminProto.PartitionEntry.newBuilder().setKey(ProtoUtils.encodeBytes(key)).setVersioned(ProtoUtils.encodeVersioned(value)).build();
        VAdminProto.UpdatePartitionEntriesRequest.Builder updateRequest = null;
        if (overWriteIfLatestTs) {
            updateRequest = VAdminProto.UpdatePartitionEntriesRequest.newBuilder().setStore(storeName).setPartitionEntry(partitionEntry).setOverwriteIfLatestTs(overWriteIfLatestTs);
        } else {
            updateRequest = VAdminProto.UpdatePartitionEntriesRequest.newBuilder().setStore(storeName).setPartitionEntry(partitionEntry);
        }
        DataOutputStream outputStream = nodeIdStoreToOutputStreamRequest.get(new Pair(storeName, node.getId()));
        try {
            if (nodeIdStoreInitialized.get(new Pair(storeName, node.getId()))) {
                ProtoUtils.writeMessage(outputStream, updateRequest.build());
            } else {
                ProtoUtils.writeMessage(outputStream, VAdminProto.VoldemortAdminRequest.newBuilder().setType(VAdminProto.AdminRequestType.UPDATE_PARTITION_ENTRIES).setUpdatePartitionEntries(updateRequest).build());
                outputStream.flush();
                nodeIdStoreInitialized.put(new Pair(storeName, node.getId()), true);
            }
        } catch (IOException e) {
            e.printStackTrace();
            nodesWithException++;
            if (!faultyNodes.contains(node.getId()))
                faultyNodes.add(node.getId());
        }
    }
    if (nodesWithException > 0) {
        logger.warn("Invoking the Recovery Callback");
        Future future = streamingresults.submit(recoveryCallback);
        try {
            future.get();
        } catch (InterruptedException e1) {
            MARKED_BAD = true;
            logger.error("Recovery Callback failed", e1);
            throw new VoldemortException("Recovery Callback failed");
        } catch (ExecutionException e1) {
            MARKED_BAD = true;
            logger.error("Recovery Callback failed during execution", e1);
            throw new VoldemortException("Recovery Callback failed during execution");
        }
    } else {
        entriesProcessed++;
        if (entriesProcessed == CHECKPOINT_COMMIT_SIZE) {
            entriesProcessed = 0;
            commitToVoldemort();
        }
        throttler.maybeThrottle(1);
    }
}

Example 20 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class RebalanceController method getCurrentClusterState.

/**
     * Probe the existing cluster to retrieve the current cluster xml and stores
     * xml.
     * 
     * @return Pair of Cluster and List<StoreDefinition> from current cluster.
     */
private Pair<Cluster, List<StoreDefinition>> getCurrentClusterState() {
    // Retrieve the latest cluster metadata from the existing nodes
    Versioned<Cluster> currentVersionedCluster = adminClient.rebalanceOps.getLatestCluster(Utils.nodeListToNodeIdList(Lists.newArrayList(adminClient.getAdminClientCluster().getNodes())));
    Cluster cluster = currentVersionedCluster.getValue();
    List<StoreDefinition> storeDefs = adminClient.rebalanceOps.getCurrentStoreDefinitions(cluster);
    return new Pair<Cluster, List<StoreDefinition>>(cluster, storeDefs);
}