Examples with UnreachableStoreException voldemort.store.UnreachableStoreException used on opensource projects

Example 26 with UnreachableStoreException

use of voldemort.store.UnreachableStoreException in project voldemort by voldemort.

the class NioSelectorManagerTest method testSelectorHeartBeat.

@Test
public void testSelectorHeartBeat() throws Exception {
    // Open a request so that Sleepy store hangs.
    pool.submitAsync(dest1, getClientRequest(), getCallBack(), 250, "get");
    // wait for MAX heart beat + some more time so that the selector
    // threads should be marked dead
    Thread.sleep(MAX_HEART_BEAT_MS + 50);
    // request, if not it calls proceesLoop which creates one more.
    try {
        pool.checkout(dest1);
    } catch (Exception ex) {
    }
    // here sleeps for infinite time
    for (int i = 0; i < 20; i++) {
        try {
            ClientRequestExecutor executor = pool.checkout(dest1);
            if (numSelectors == 1) {
                fail("Single selector expected an exception");
            } else {
                assertNotNull("multiple selector should have valid executor", executor);
            }
        } catch (UnreachableStoreException ex) {
            if (numSelectors > 1) {
                fail("more than one selector, no exception should be observed" + ex);
            } else {
                assertTrue("inner exception should be instance of IOException", ex.getCause() instanceof IOException);
            }
        }
    }
    sleepyStore.releaseThreads();
}

Example 27 with UnreachableStoreException

use of voldemort.store.UnreachableStoreException in project voldemort by voldemort.

the class RoutedStoreTest method testBasicOperationFailureZZZMultiThreaded.

/**
     * Test to ensure that the basic operations fail in the presence of bad
     * nodes in a 3 zone cluster.
     * 
     * @throws Exception
     */
@Test
public void testBasicOperationFailureZZZMultiThreaded() throws Exception {
    cluster = VoldemortTestConstants.getNineNodeClusterWith3Zones();
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(1, cluster.getNumberOfNodesInZone(0));
    zoneReplicationFactor.put(2, cluster.getNumberOfNodesInZone(0));
    // PR = RR = 7
    // PW = RW = 7
    // Zone Reads = # Zones - 1
    // Zone Writes = # Zones - 1
    // Failing nodes = 1 from each zone
    // Threads = 4
    RoutedStore zonedRoutedStore = getStore(cluster, 7, 7, cluster.getNumberOfZones() - 1, cluster.getNumberOfZones() - 1, 4, Sets.newHashSet(1, 5, 6), null, zoneReplicationFactor, RoutingStrategyType.ZONE_STRATEGY, 0, BANNAGE_PERIOD, new UnreachableStoreException("no go"));
    testBasicOperationFailure(cluster.getNumberOfNodes() - 2, cluster.getNumberOfNodes() - 2, 0, 0, zonedRoutedStore);
}

Example 28 with UnreachableStoreException

use of voldemort.store.UnreachableStoreException in project voldemort by voldemort.

the class ThreadPoolRoutedStore method put.

@Override
public void put(final ByteArray key, final Versioned<byte[]> versioned, final byte[] transforms) throws VoldemortException {
    long startNs = System.nanoTime();
    StoreUtils.assertValidKey(key);
    final List<Node> nodes = availableNodes(routingStrategy.routeRequest(key.get()));
    // quickly fail if there aren't enough nodes to meet the requirement
    final int numNodes = nodes.size();
    if (numNodes < this.storeDef.getRequiredWrites())
        throw new InsufficientOperationalNodesException("Only " + numNodes + " nodes in preference list, but " + this.storeDef.getRequiredWrites() + " writes required.");
    // A count of the number of successful operations
    final AtomicInteger successes = new AtomicInteger(0);
    // A list of thrown exceptions, indicating the number of failures
    final List<Exception> failures = new CopyOnWriteArrayList<Exception>();
    // If requiredWrites > 0 then do a single blocking write to the first
    // live node in the preference list if this node throws an
    // ObsoleteVersionException allow it to propagate
    Node master = null;
    int currentNode = 0;
    Versioned<byte[]> versionedCopy = null;
    for (; currentNode < numNodes; currentNode++) {
        Node current = nodes.get(currentNode);
        long startNsLocal = System.nanoTime();
        try {
            versionedCopy = incremented(versioned, current.getId());
            innerStores.get(current.getId()).put(key, versionedCopy, transforms);
            successes.getAndIncrement();
            recordSuccess(current, startNsLocal);
            master = current;
            break;
        } catch (UnreachableStoreException e) {
            recordException(current, startNsLocal, e);
            failures.add(e);
        } catch (VoldemortApplicationException e) {
            throw e;
        } catch (Exception e) {
            failures.add(e);
        }
    }
    if (successes.get() < 1)
        throw new InsufficientOperationalNodesException("No master node succeeded!", failures);
    else
        currentNode++;
    // A semaphore indicating the number of completed operations
    // Once inititialized all permits are acquired, after that
    // permits are released when an operation is completed.
    // semaphore.acquire(n) waits for n operations to complete
    final Versioned<byte[]> finalVersionedCopy = versionedCopy;
    final Semaphore semaphore = new Semaphore(0, false);
    // Add the operations to the pool
    int attempts = 0;
    for (; currentNode < numNodes; currentNode++) {
        attempts++;
        final Node node = nodes.get(currentNode);
        this.executor.execute(new Runnable() {

            @Override
            public void run() {
                long startNsLocal = System.nanoTime();
                try {
                    innerStores.get(node.getId()).put(key, finalVersionedCopy, transforms);
                    successes.incrementAndGet();
                    recordSuccess(node, startNsLocal);
                } catch (UnreachableStoreException e) {
                    recordException(node, startNsLocal, e);
                    failures.add(e);
                } catch (ObsoleteVersionException e) {
                // ignore this completely here
                // this means that a higher version was able
                // to write on this node and should be termed as clean
                // success.
                } catch (VoldemortApplicationException e) {
                    throw e;
                } catch (Exception e) {
                    logger.warn("Error in PUT on node " + node.getId() + "(" + node.getHost() + ")", e);
                    failures.add(e);
                } finally {
                    // signal that the operation is complete
                    semaphore.release();
                }
            }
        });
    }
    // Block until we get enough completions
    int blockCount = Math.min(storeDef.getPreferredWrites() - 1, attempts);
    boolean noTimeout = blockOnPut(startNs, semaphore, 0, blockCount, successes, storeDef.getPreferredWrites());
    if (successes.get() < storeDef.getRequiredWrites()) {
        /*
             * We don't have enough required writes, but we haven't timed out
             * yet, so block a little more if there are healthy nodes that can
             * help us achieve our target.
             */
        if (noTimeout) {
            int startingIndex = blockCount - 1;
            blockCount = Math.max(storeDef.getPreferredWrites() - 1, attempts);
            blockOnPut(startNs, semaphore, startingIndex, blockCount, successes, storeDef.getRequiredWrites());
        }
        if (successes.get() < storeDef.getRequiredWrites())
            throw new InsufficientOperationalNodesException(successes.get() + " writes succeeded, but " + this.storeDef.getRequiredWrites() + " are required.", failures);
    }
    // Okay looks like it worked, increment the version for the caller
    VectorClock versionedClock = (VectorClock) versioned.getVersion();
    versionedClock.incrementVersion(master.getId(), time.getMilliseconds());
}

Example 29 with UnreachableStoreException

use of voldemort.store.UnreachableStoreException in project voldemort by voldemort.

the class HintedHandoff method sendOneAsyncHint.

/**
     * A callback that handles requestComplete event from NIO selector manager
     * Will try any possible nodes and pass itself as callback util all nodes
     * are exhausted
     * 
     * @param slopKey
     * @param slopVersioned
     * @param nodesToTry List of nodes to try to contact. Will become shorter
     *        after each callback
     */
private void sendOneAsyncHint(final ByteArray slopKey, final Versioned<byte[]> slopVersioned, final List<Node> nodesToTry) {
    Node nodeToHostHint = null;
    boolean foundNode = false;
    while (nodesToTry.size() > 0) {
        nodeToHostHint = nodesToTry.remove(0);
        if (!failedNodes.contains(nodeToHostHint) && failureDetector.isAvailable(nodeToHostHint)) {
            foundNode = true;
            break;
        }
    }
    if (!foundNode) {
        Slop slop = slopSerializer.toObject(slopVersioned.getValue());
        logger.error("Trying to send an async hint but used up all nodes. key: " + slop.getKey() + " version: " + slopVersioned.getVersion().toString());
        return;
    }
    final Node node = nodeToHostHint;
    int nodeId = node.getId();
    NonblockingStore nonblockingStore = nonblockingSlopStores.get(nodeId);
    Utils.notNull(nonblockingStore);
    final Long startNs = System.nanoTime();
    NonblockingStoreCallback callback = new NonblockingStoreCallback() {

        @Override
        public void requestComplete(Object result, long requestTime) {
            Slop slop = null;
            boolean loggerDebugEnabled = logger.isDebugEnabled();
            if (loggerDebugEnabled) {
                slop = slopSerializer.toObject(slopVersioned.getValue());
            }
            Response<ByteArray, Object> response = new Response<ByteArray, Object>(node, slopKey, result, requestTime);
            if (response.getValue() instanceof Exception && !(response.getValue() instanceof ObsoleteVersionException)) {
                if (!failedNodes.contains(node))
                    failedNodes.add(node);
                if (response.getValue() instanceof UnreachableStoreException) {
                    UnreachableStoreException use = (UnreachableStoreException) response.getValue();
                    if (loggerDebugEnabled) {
                        logger.debug("Write of key " + slop.getKey() + " for " + slop.getNodeId() + " to node " + node + " failed due to unreachable: " + use.getMessage());
                    }
                    failureDetector.recordException(node, (System.nanoTime() - startNs) / Time.NS_PER_MS, use);
                }
                sendOneAsyncHint(slopKey, slopVersioned, nodesToTry);
            }
            if (loggerDebugEnabled)
                logger.debug("Slop write of key " + slop.getKey() + " for node " + slop.getNodeId() + " to node " + node + " succeeded in " + (System.nanoTime() - startNs) + " ns");
            failureDetector.recordSuccess(node, (System.nanoTime() - startNs) / Time.NS_PER_MS);
        }
    };
    nonblockingStore.submitPutRequest(slopKey, slopVersioned, null, callback, timeoutMs);
}

Example 30 with UnreachableStoreException

use of voldemort.store.UnreachableStoreException in project voldemort by voldemort.

the class HintedHandoff method sendHintSerial.

/**
     * Send a hint of a request originally meant for the failed node to another
     * node in the ring, as selected by the {@link HintedHandoffStrategy}
     * implementation passed in the constructor
     * 
     * @param failedNode The node the request was originally meant for
     * @param version The version of the request's object
     * @param slop The hint
     * @return True if persisted on another node, false otherwise
     */
@Deprecated
public boolean sendHintSerial(Node failedNode, Version version, Slop slop) {
    boolean persisted = false;
    for (Node node : handoffStrategy.routeHint(failedNode)) {
        int nodeId = node.getId();
        if (!failedNodes.contains(node) && failureDetector.isAvailable(node)) {
            if (logger.isDebugEnabled())
                logger.debug("Trying to send hint to " + nodeId + " for key " + slop.getKey());
            Store<ByteArray, Slop, byte[]> slopStore = slopStores.get(nodeId);
            Utils.notNull(slopStore);
            long startNs = System.nanoTime();
            try {
                if (logger.isDebugEnabled())
                    logger.debug("Slop attempt to write " + slop.getKey() + " (keyRef: " + System.identityHashCode(slop.getKey()) + ") for " + failedNode + " to node " + node);
                // No transform needs to applied to the slop
                slopStore.put(slop.makeKey(), new Versioned<Slop>(slop, version), null);
                persisted = true;
                failureDetector.recordSuccess(node, (System.nanoTime() - startNs) / Time.NS_PER_MS);
                if (logger.isTraceEnabled())
                    logger.trace("Finished hinted handoff for " + failedNode + " wrote slop to " + node);
                break;
            } catch (UnreachableStoreException e) {
                failureDetector.recordException(node, (System.nanoTime() - startNs) / Time.NS_PER_MS, e);
                logger.warn("Error during hinted handoff. Will try another node", e);
            } catch (IllegalStateException e) {
                logger.warn("Error during hinted handoff. Will try another node", e);
            } catch (ObsoleteVersionException e) {
                logger.debug(e, e);
            } catch (Exception e) {
                logger.error("Unknown exception. Will try another node" + e);
            }
            if (logger.isDebugEnabled())
                logger.debug("Slop write of key " + slop.getKey() + " (keyRef: " + System.identityHashCode(slop.getKey()) + ") for " + failedNode + " to node " + node + (persisted ? " succeeded" : " failed") + " in " + (System.nanoTime() - startNs) + " ns");
        } else {
            if (logger.isDebugEnabled()) {
                logger.debug("Skipping node " + nodeId);
            }
        }
    }
    if (!persisted) {
        logger.error("Slop write of key " + slop.getKey() + " (keyRef: " + System.identityHashCode(slop.getKey()) + ") for " + failedNode + " was not written.");
    }
    return persisted;
}