Examples with InsufficientOperationalNodesException voldemort.store.InsufficientOperationalNodesException used on opensource projects

Example 16 with InsufficientOperationalNodesException

use of voldemort.store.InsufficientOperationalNodesException in project voldemort by voldemort.

the class ZoneCountWriteTest method testRemoteZoneNodeFail.

@Test
public void testRemoteZoneNodeFail() {
    try {
        for (Integer nodeId : stoppedServersForRemoteZoneNodeFail) {
            vservers.get(nodeId).stop();
        }
        client.put("AB", new Versioned<String>("CD"), null);
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
        }
        for (Integer nodeId : vservers.keySet()) {
            // skip stopped ones
            if (stoppedServersForRemoteZoneNodeFail.contains(nodeId)) {
                continue;
            }
            VoldemortServer vs = vservers.get(nodeId);
            Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getLocalStore(storeDef.getName());
            byte[] real = store.get(new ByteArray("AB".getBytes()), null).get(0).getValue();
            assertTrue(Arrays.equals(real, "CD".getBytes()));
        }
    } catch (InsufficientOperationalNodesException e) {
        fail("Failed with exception: " + e);
        e.printStackTrace();
    }
}

Example 17 with InsufficientOperationalNodesException

use of voldemort.store.InsufficientOperationalNodesException in project voldemort by voldemort.

the class LongHintedHandoffTest method testHintedHandoff.

@Test
public void testHintedHandoff() throws InterruptedException {
    Set<Integer> nodeIds = new HashSet<Integer>();
    long startMs = System.currentTimeMillis();
    long endMs = startMs + MAX_TOTAL_TIME_MS;
    long totalPuts = 0;
    long numRejectedPuts = 0;
    long numAssertPuts = 0;
    StoreClient<byte[], byte[]> client = testEnv.makeClient();
    while (true) {
        if (System.currentTimeMillis() > endMs) {
            break;
        }
        // generate key
        ByteArray key = new ByteArray(TestUtils.randomBytes(KEY_LENGTH));
        byte[] value = TestUtils.randomBytes(VALUE_LENGTH);
        // put to nodes
        try {
            client.put(key.get(), value);
            // if put does not throw exception
            List<Node> routes = testEnv.routeRequest(key.get());
            for (Node node : routes) {
                numAssertPuts++;
                nodeIds.add(node.getId());
                Store<ByteArray, byte[], byte[]> realStore = testEnv.getRealStore(node.getId());
                if (realStore instanceof InMemoryPutAssertionStorageEngine) {
                    ((InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) realStore).assertPut(key);
                } else {
                    fail("realStore is not InMemoryPutAssertionStorageEngine. Test setup is wrong");
                }
            }
        } catch (InsufficientOperationalNodesException e) {
            numRejectedPuts++;
            if (logger.isDebugEnabled()) {
                logger.debug("Key " + key + " is rejected for InsufficientOperationalNodes");
            }
        } finally {
            totalPuts++;
        }
    }
    // bring all servers up
    testEnv.warpUp();
    // check
    long numFailedAssertions = 0;
    for (Integer nodeId : nodeIds) {
        Store<ByteArray, byte[], byte[]> realStore = testEnv.getRealStore(nodeId);
        if (realStore instanceof InMemoryPutAssertionStorageEngine) {
            Set<ByteArray> keys = ((InMemoryPutAssertionStorageEngine<ByteArray, byte[], byte[]>) realStore).getFailedAssertions();
            for (ByteArray key : keys) {
                logger.error("key [" + key + "] is asserted but not recorded on node [" + nodeId + "]");
            }
            numFailedAssertions += keys.size();
        } else {
            fail("realStore is not InMemoryPutAssertionStorageEngine");
        }
    }
    logger.info("Total Client Puts Rejected (InsufficientOperationalNodes): " + numRejectedPuts);
    logger.info("Total Client Put Operations: " + totalPuts);
    logger.info("Total Server Put Assertions: " + numAssertPuts);
    logger.info("Total Server Put Lost: " + numFailedAssertions);
    assertEquals(numFailedAssertions + " server puts are lost: " + numFailedAssertions, 0L, numFailedAssertions);
}

Example 18 with InsufficientOperationalNodesException

use of voldemort.store.InsufficientOperationalNodesException in project voldemort by voldemort.

the class RoutedStoreTest method testOnlyNodeFailuresDisableNode.

@Test
public void testOnlyNodeFailuresDisableNode() throws Exception {
    // test put
    cluster = getNineNodeCluster();
    Store<ByteArray, byte[], byte[]> s1 = getStore(cluster, 1, 9, 9, 9, 0, RoutingStrategyType.TO_ALL_STRATEGY, new VoldemortException());
    try {
        s1.put(aKey, new Versioned<byte[]>(aValue), aTransform);
        fail("Failure is expected");
    } catch (InsufficientOperationalNodesException e) {
    /* expected */
    }
    assertOperationalNodes(9);
    cluster = getNineNodeCluster();
    Store<ByteArray, byte[], byte[]> s2 = getStore(cluster, 1, 9, 9, 9, 0, RoutingStrategyType.TO_ALL_STRATEGY, new UnreachableStoreException("no go"));
    try {
        s2.put(aKey, new Versioned<byte[]>(aValue), aTransform);
        fail("Failure is expected");
    } catch (InsufficientOperationalNodesException e) {
    /* expected */
    }
    assertOperationalNodes(0);
    // test get
    cluster = getNineNodeCluster();
    s1 = getStore(cluster, 1, 9, 9, 9, 0, RoutingStrategyType.TO_ALL_STRATEGY, new VoldemortException());
    try {
        s1.get(aKey, aTransform);
        fail("Failure is expected");
    } catch (InsufficientOperationalNodesException e) {
    /* expected */
    }
    assertOperationalNodes(9);
    cluster = getNineNodeCluster();
    s2 = getStore(cluster, 1, 9, 9, 9, 0, RoutingStrategyType.TO_ALL_STRATEGY, new UnreachableStoreException("no go"));
    try {
        s2.get(aKey, aTransform);
        fail("Failure is expected");
    } catch (InsufficientOperationalNodesException e) {
    /* expected */
    }
    assertOperationalNodes(0);
    // test delete
    cluster = getNineNodeCluster();
    s1 = getStore(cluster, 1, 9, 9, 9, 0, RoutingStrategyType.TO_ALL_STRATEGY, new VoldemortException());
    try {
        s1.delete(aKey, new VectorClock());
        fail("Failure is expected");
    } catch (InsufficientOperationalNodesException e) {
    /* expected */
    }
    assertOperationalNodes(9);
    cluster = getNineNodeCluster();
    s2 = getStore(cluster, 1, 9, 9, 9, 0, RoutingStrategyType.TO_ALL_STRATEGY, new UnreachableStoreException("no go"));
    try {
        s2.delete(aKey, new VectorClock());
        fail("Failure is expected");
    } catch (InsufficientOperationalNodesException e) {
    /* expected */
    }
    assertOperationalNodes(0);
}

Example 19 with InsufficientOperationalNodesException

use of voldemort.store.InsufficientOperationalNodesException in project voldemort by voldemort.

the class BaseStreamingClient method validateSufficientNodesAvailable.

private void validateSufficientNodesAvailable(List<Integer> blackListedNodes, StoreDefinition remoteStoreDef) {
    int faultyNodes = 0;
    if (blackListedNodes != null && blackListedNodes.size() > 0) {
        faultyNodes = blackListedNodes.size();
    }
    int repFactor = remoteStoreDef.getReplicationFactor();
    int numNodes = repFactor - faultyNodes;
    if (numNodes < remoteStoreDef.getRequiredWrites())
        throw new InsufficientOperationalNodesException("Only " + numNodes + " nodes in preference list, but " + remoteStoreDef.getRequiredWrites() + " writes required.");
}

Example 20 with InsufficientOperationalNodesException

use of voldemort.store.InsufficientOperationalNodesException in project voldemort by voldemort.

the class ThreadPoolRoutedStore method put.

@Override
public void put(final ByteArray key, final Versioned<byte[]> versioned, final byte[] transforms) throws VoldemortException {
    long startNs = System.nanoTime();
    StoreUtils.assertValidKey(key);
    final List<Node> nodes = availableNodes(routingStrategy.routeRequest(key.get()));
    // quickly fail if there aren't enough nodes to meet the requirement
    final int numNodes = nodes.size();
    if (numNodes < this.storeDef.getRequiredWrites())
        throw new InsufficientOperationalNodesException("Only " + numNodes + " nodes in preference list, but " + this.storeDef.getRequiredWrites() + " writes required.");
    // A count of the number of successful operations
    final AtomicInteger successes = new AtomicInteger(0);
    // A list of thrown exceptions, indicating the number of failures
    final List<Exception> failures = new CopyOnWriteArrayList<Exception>();
    // If requiredWrites > 0 then do a single blocking write to the first
    // live node in the preference list if this node throws an
    // ObsoleteVersionException allow it to propagate
    Node master = null;
    int currentNode = 0;
    Versioned<byte[]> versionedCopy = null;
    for (; currentNode < numNodes; currentNode++) {
        Node current = nodes.get(currentNode);
        long startNsLocal = System.nanoTime();
        try {
            versionedCopy = incremented(versioned, current.getId());
            innerStores.get(current.getId()).put(key, versionedCopy, transforms);
            successes.getAndIncrement();
            recordSuccess(current, startNsLocal);
            master = current;
            break;
        } catch (UnreachableStoreException e) {
            recordException(current, startNsLocal, e);
            failures.add(e);
        } catch (VoldemortApplicationException e) {
            throw e;
        } catch (Exception e) {
            failures.add(e);
        }
    }
    if (successes.get() < 1)
        throw new InsufficientOperationalNodesException("No master node succeeded!", failures);
    else
        currentNode++;
    // A semaphore indicating the number of completed operations
    // Once inititialized all permits are acquired, after that
    // permits are released when an operation is completed.
    // semaphore.acquire(n) waits for n operations to complete
    final Versioned<byte[]> finalVersionedCopy = versionedCopy;
    final Semaphore semaphore = new Semaphore(0, false);
    // Add the operations to the pool
    int attempts = 0;
    for (; currentNode < numNodes; currentNode++) {
        attempts++;
        final Node node = nodes.get(currentNode);
        this.executor.execute(new Runnable() {

            @Override
            public void run() {
                long startNsLocal = System.nanoTime();
                try {
                    innerStores.get(node.getId()).put(key, finalVersionedCopy, transforms);
                    successes.incrementAndGet();
                    recordSuccess(node, startNsLocal);
                } catch (UnreachableStoreException e) {
                    recordException(node, startNsLocal, e);
                    failures.add(e);
                } catch (ObsoleteVersionException e) {
                // ignore this completely here
                // this means that a higher version was able
                // to write on this node and should be termed as clean
                // success.
                } catch (VoldemortApplicationException e) {
                    throw e;
                } catch (Exception e) {
                    logger.warn("Error in PUT on node " + node.getId() + "(" + node.getHost() + ")", e);
                    failures.add(e);
                } finally {
                    // signal that the operation is complete
                    semaphore.release();
                }
            }
        });
    }
    // Block until we get enough completions
    int blockCount = Math.min(storeDef.getPreferredWrites() - 1, attempts);
    boolean noTimeout = blockOnPut(startNs, semaphore, 0, blockCount, successes, storeDef.getPreferredWrites());
    if (successes.get() < storeDef.getRequiredWrites()) {
        /*
             * We don't have enough required writes, but we haven't timed out
             * yet, so block a little more if there are healthy nodes that can
             * help us achieve our target.
             */
        if (noTimeout) {
            int startingIndex = blockCount - 1;
            blockCount = Math.max(storeDef.getPreferredWrites() - 1, attempts);
            blockOnPut(startNs, semaphore, startingIndex, blockCount, successes, storeDef.getRequiredWrites());
        }
        if (successes.get() < storeDef.getRequiredWrites())
            throw new InsufficientOperationalNodesException(successes.get() + " writes succeeded, but " + this.storeDef.getRequiredWrites() + " are required.", failures);
    }
    // Okay looks like it worked, increment the version for the caller
    VectorClock versionedClock = (VectorClock) versioned.getVersion();
    versionedClock.incrementVersion(master.getId(), time.getMilliseconds());
}