Search in sources :

Example 6 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class RestServiceR2StoreTest method testGetVersions.

@Override
@Test
public void testGetVersions() throws Exception {
    List<ByteArray> keys = getKeys(2);
    ByteArray key = keys.get(0);
    byte[] value = getValue();
    VectorClock vc = getClock(0, 0);
    Store<ByteArray, byte[], byte[]> store = getStore();
    store.put(key, Versioned.value(value, vc), null);
    List<Versioned<byte[]>> versioneds = store.get(key, null);
    List<Version> versions = store.getVersions(key);
    assertEquals(1, versioneds.size());
    assertTrue(versions.size() > 0);
    for (int i = 0; i < versions.size(); i++) assertEquals(versioneds.get(0).getVersion(), versions.get(i));
    assertEquals(0, store.getVersions(keys.get(1)).size());
}
Also used : Versioned(voldemort.versioning.Versioned) Version(voldemort.versioning.Version) VectorClock(voldemort.versioning.VectorClock) ByteArray(voldemort.utils.ByteArray) AbstractByteArrayStoreTest(voldemort.store.AbstractByteArrayStoreTest) Test(org.junit.Test)

Example 7 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class RestServiceR2StoreTest method testNullKeys.

@Override
@Test
public void testNullKeys() throws Exception {
    System.out.println("                    Testing null Keys                    ");
    Store<ByteArray, byte[], byte[]> store = getStore();
    try {
        store.put(null, new Versioned<byte[]>(getValue(), getNewIncrementedVectorClock()), null);
        fail("Store should not put null keys!");
    } catch (IllegalArgumentException e) {
    // this is good
    } catch (NullPointerException npe) {
    // this is good
    }
    try {
        store.get(null, null);
        fail("Store should not get null keys!");
    } catch (IllegalArgumentException e) {
    // this is good
    } catch (NullPointerException npe) {
    // this is good
    }
    try {
        store.getAll(null, null);
        fail("Store should not getAll null keys!");
    } catch (IllegalArgumentException e) {
    // this is good
    } catch (NullPointerException npe) {
    // this is good
    }
    try {
        store.getAll(Collections.<ByteArray>singleton(null), Collections.<ByteArray, byte[]>singletonMap(null, null));
        fail("Store should not getAll null keys!");
    } catch (IllegalArgumentException e) {
    // this is good
    } catch (NullPointerException npe) {
    // this is good
    }
    try {
        store.delete(null, new VectorClock());
        fail("Store should not delete null keys!");
    } catch (IllegalArgumentException e) {
    // this is good
    } catch (NullPointerException npe) {
    // this is good
    }
}
Also used : VectorClock(voldemort.versioning.VectorClock) ByteArray(voldemort.utils.ByteArray) AbstractByteArrayStoreTest(voldemort.store.AbstractByteArrayStoreTest) Test(org.junit.Test)

Example 8 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class PipelineRoutedStore method getAll.

public Map<ByteArray, List<Versioned<byte[]>>> getAll(Iterable<ByteArray> keys, Map<ByteArray, byte[]> transforms, long getAllOpTimeoutInMs) throws VoldemortException {
    StoreUtils.assertValidKeys(keys);
    long startTimeMs = -1;
    long startTimeNs = -1;
    if (logger.isDebugEnabled()) {
        startTimeMs = System.currentTimeMillis();
        startTimeNs = System.nanoTime();
    }
    boolean allowReadRepair = repairReads && (transforms == null || transforms.size() == 0);
    GetAllPipelineData pipelineData = new GetAllPipelineData();
    if (zoneRoutingEnabled)
        pipelineData.setZonesRequired(storeDef.getZoneCountReads());
    else
        pipelineData.setZonesRequired(null);
    pipelineData.setStats(stats);
    Pipeline pipeline = new Pipeline(Operation.GET_ALL, getAllOpTimeoutInMs, TimeUnit.MILLISECONDS);
    pipeline.addEventAction(Event.STARTED, new GetAllConfigureNodes(pipelineData, Event.CONFIGURED, failureDetector, storeDef.getPreferredReads(), storeDef.getRequiredReads(), routingStrategy, keys, transforms, clientZone, zoneAffinity));
    pipeline.addEventAction(Event.CONFIGURED, new PerformParallelGetAllRequests(pipelineData, Event.INSUFFICIENT_SUCCESSES, failureDetector, getAllOpTimeoutInMs, nonblockingStores));
    pipeline.addEventAction(Event.INSUFFICIENT_SUCCESSES, new PerformSerialGetAllRequests(pipelineData, allowReadRepair ? Event.RESPONSES_RECEIVED : Event.COMPLETED, keys, failureDetector, innerStores, storeDef.getPreferredReads(), storeDef.getRequiredReads(), timeoutConfig.isPartialGetAllAllowed()));
    if (allowReadRepair)
        pipeline.addEventAction(Event.RESPONSES_RECEIVED, new GetAllReadRepair(pipelineData, Event.COMPLETED, storeDef.getPreferredReads(), getAllOpTimeoutInMs, nonblockingStores, readRepairer));
    pipeline.addEvent(Event.STARTED);
    if (logger.isDebugEnabled()) {
        StringBuilder keyStr = new StringBuilder();
        for (ByteArray key : keys) {
            keyStr.append(ByteUtils.toHexString(key.get()) + ",");
        }
        logger.debug("Operation " + pipeline.getOperation().getSimpleName() + " Keys " + keyStr.toString());
    }
    try {
        pipeline.execute();
    } catch (VoldemortException e) {
        stats.reportException(e);
        throw e;
    }
    if (pipelineData.getFatalError() != null)
        throw pipelineData.getFatalError();
    if (logger.isDebugEnabled()) {
        logger.debug("Finished " + pipeline.getOperation().getSimpleName() + "for keys " + ByteArray.toHexStrings(keys) + " keyRef: " + System.identityHashCode(keys) + "; started at " + startTimeMs + " took " + (System.nanoTime() - startTimeNs) + " values: " + formatNodeValuesFromGetAll(pipelineData.getResponses()));
    }
    return pipelineData.getResult();
}
Also used : GetAllReadRepair(voldemort.store.routed.action.GetAllReadRepair) PerformSerialGetAllRequests(voldemort.store.routed.action.PerformSerialGetAllRequests) GetAllConfigureNodes(voldemort.store.routed.action.GetAllConfigureNodes) ByteArray(voldemort.utils.ByteArray) PerformParallelGetAllRequests(voldemort.store.routed.action.PerformParallelGetAllRequests) VoldemortException(voldemort.VoldemortException)

Example 9 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class ThreadPoolRoutedStore method getAll.

@Override
public Map<ByteArray, List<Versioned<byte[]>>> getAll(Iterable<ByteArray> keys, Map<ByteArray, byte[]> transforms) throws VoldemortException {
    StoreUtils.assertValidKeys(keys);
    Map<ByteArray, List<Versioned<byte[]>>> result = StoreUtils.newEmptyHashMap(keys);
    // Keys for each node needed to satisfy storeDef.getPreferredReads() if
    // no failures.
    Map<Node, List<ByteArray>> nodeToKeysMap = Maps.newHashMap();
    // Keep track of nodes per key that might be needed if there are
    // failures during getAll
    Map<ByteArray, List<Node>> keyToExtraNodesMap = Maps.newHashMap();
    for (ByteArray key : keys) {
        List<Node> availableNodes = availableNodes(routingStrategy.routeRequest(key.get()));
        // quickly fail if there aren't enough nodes to meet the requirement
        checkRequiredReads(availableNodes);
        int preferredReads = storeDef.getPreferredReads();
        List<Node> preferredNodes = Lists.newArrayListWithCapacity(preferredReads);
        List<Node> extraNodes = Lists.newArrayListWithCapacity(3);
        for (Node node : availableNodes) {
            if (preferredNodes.size() < preferredReads)
                preferredNodes.add(node);
            else
                extraNodes.add(node);
        }
        for (Node node : preferredNodes) {
            List<ByteArray> nodeKeys = nodeToKeysMap.get(node);
            if (nodeKeys == null) {
                nodeKeys = Lists.newArrayList();
                nodeToKeysMap.put(node, nodeKeys);
            }
            nodeKeys.add(key);
        }
        if (!extraNodes.isEmpty()) {
            List<Node> nodes = keyToExtraNodesMap.get(key);
            if (nodes == null)
                keyToExtraNodesMap.put(key, extraNodes);
            else
                nodes.addAll(extraNodes);
        }
    }
    List<Callable<GetAllResult>> callables = Lists.newArrayList();
    for (Map.Entry<Node, List<ByteArray>> entry : nodeToKeysMap.entrySet()) {
        final Node node = entry.getKey();
        final Collection<ByteArray> nodeKeys = entry.getValue();
        if (failureDetector.isAvailable(node))
            callables.add(new GetAllCallable(node, nodeKeys, transforms));
    }
    // A list of thrown exceptions, indicating the number of failures
    List<Throwable> failures = new CopyOnWriteArrayList<Throwable>();
    List<NodeValue<ByteArray, byte[]>> nodeValues = Lists.newArrayList();
    Map<ByteArray, MutableInt> keyToSuccessCount = Maps.newHashMap();
    for (ByteArray key : keys) keyToSuccessCount.put(key, new MutableInt(0));
    List<Future<GetAllResult>> futures;
    long timeoutMs = timeoutConfig.getOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE);
    try {
        // TODO What to do about timeouts? They should be longer as getAll
        // is likely to
        // take longer. At the moment, it's just timeoutMs * 3, but should
        // this be based on the number of the keys?
        futures = executor.invokeAll(callables, timeoutMs * 3, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        throw new InsufficientOperationalNodesException("getAll operation interrupted.", e);
    }
    for (Future<GetAllResult> f : futures) {
        if (f.isCancelled()) {
            logger.warn("Get operation timed out after " + timeoutMs + " ms.");
            continue;
        }
        try {
            GetAllResult getResult = f.get();
            if (getResult.exception != null) {
                if (getResult.exception instanceof VoldemortApplicationException) {
                    throw (VoldemortException) getResult.exception;
                }
                failures.add(getResult.exception);
                continue;
            }
            for (ByteArray key : getResult.callable.nodeKeys) {
                List<Versioned<byte[]>> retrieved = getResult.retrieved.get(key);
                MutableInt successCount = keyToSuccessCount.get(key);
                successCount.increment();
                /*
                     * retrieved can be null if there are no values for the key
                     * provided
                     */
                if (retrieved != null) {
                    List<Versioned<byte[]>> existing = result.get(key);
                    if (existing == null)
                        result.put(key, Lists.newArrayList(retrieved));
                    else
                        existing.addAll(retrieved);
                }
            }
            nodeValues.addAll(getResult.nodeValues);
        } catch (InterruptedException e) {
            throw new InsufficientOperationalNodesException("getAll operation interrupted.", e);
        } catch (ExecutionException e) {
            // should never happen
            if (e.getCause() instanceof Error)
                throw (Error) e.getCause();
            else
                logger.error(e.getMessage(), e);
        }
    }
    for (ByteArray key : keys) {
        MutableInt successCountWrapper = keyToSuccessCount.get(key);
        int successCount = successCountWrapper.intValue();
        if (successCount < storeDef.getPreferredReads()) {
            List<Node> extraNodes = keyToExtraNodesMap.get(key);
            if (extraNodes != null) {
                for (Node node : extraNodes) {
                    long startNs = System.nanoTime();
                    try {
                        List<Versioned<byte[]>> values = innerStores.get(node.getId()).get(key, transforms == null ? null : transforms.get(key));
                        fillRepairReadsValues(nodeValues, key, node, values);
                        List<Versioned<byte[]>> versioneds = result.get(key);
                        if (versioneds == null)
                            result.put(key, Lists.newArrayList(values));
                        else
                            versioneds.addAll(values);
                        recordSuccess(node, startNs);
                        if (++successCount >= storeDef.getPreferredReads())
                            break;
                    } catch (UnreachableStoreException e) {
                        failures.add(e);
                        recordException(node, startNs, e);
                    } catch (VoldemortApplicationException e) {
                        throw e;
                    } catch (Exception e) {
                        logger.warn("Error in GET_ALL on node " + node.getId() + "(" + node.getHost() + ")", e);
                        failures.add(e);
                    }
                }
            }
        }
        successCountWrapper.setValue(successCount);
    }
    repairReads(nodeValues, repairReads && (transforms == null || transforms.size() == 0));
    for (Map.Entry<ByteArray, MutableInt> mapEntry : keyToSuccessCount.entrySet()) {
        int successCount = mapEntry.getValue().intValue();
        if (successCount < storeDef.getRequiredReads())
            throw new InsufficientOperationalNodesException(this.storeDef.getRequiredReads() + " reads required, but " + successCount + " succeeded.", failures);
    }
    return result;
}
Also used : Versioned(voldemort.versioning.Versioned) Node(voldemort.cluster.Node) VoldemortException(voldemort.VoldemortException) Callable(java.util.concurrent.Callable) ByteArray(voldemort.utils.ByteArray) ArrayList(java.util.ArrayList) List(java.util.List) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) UnreachableStoreException(voldemort.store.UnreachableStoreException) ExecutionException(java.util.concurrent.ExecutionException) VoldemortApplicationException(voldemort.VoldemortApplicationException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) InsufficientOperationalNodesException(voldemort.store.InsufficientOperationalNodesException) VoldemortException(voldemort.VoldemortException) VoldemortApplicationException(voldemort.VoldemortApplicationException) UnreachableStoreException(voldemort.store.UnreachableStoreException) ExecutionException(java.util.concurrent.ExecutionException) MutableInt(org.apache.commons.lang.mutable.MutableInt) InsufficientOperationalNodesException(voldemort.store.InsufficientOperationalNodesException) Future(java.util.concurrent.Future) Map(java.util.Map) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList)

Example 10 with ByteArray

use of voldemort.utils.ByteArray in project voldemort by voldemort.

the class AbstractReadRepair method execute.

public void execute(Pipeline pipeline) {
    insertNodeValues();
    long startTimeNs = -1;
    if (logger.isDebugEnabled())
        startTimeNs = System.nanoTime();
    if (nodeValues.size() > 1 && preferred > 1) {
        List<NodeValue<ByteArray, byte[]>> toReadRepair = Lists.newArrayList();
        /*
             * We clone after computing read repairs in the assumption that the
             * output will be smaller than the input. Note that we clone the
             * version, but not the key or value as the latter two are not
             * mutated.
             */
        for (NodeValue<ByteArray, byte[]> v : readRepairer.getRepairs(nodeValues)) {
            Versioned<byte[]> versioned = Versioned.value(v.getVersioned().getValue(), ((VectorClock) v.getVersion()).clone());
            toReadRepair.add(new NodeValue<ByteArray, byte[]>(v.getNodeId(), v.getKey(), versioned));
        }
        for (NodeValue<ByteArray, byte[]> v : toReadRepair) {
            try {
                if (logger.isDebugEnabled())
                    logger.debug("Doing read repair on node " + v.getNodeId() + " for key '" + ByteUtils.toHexString(v.getKey().get()) + "' with version " + v.getVersion() + ".");
                NonblockingStore store = nonblockingStores.get(v.getNodeId());
                store.submitPutRequest(v.getKey(), v.getVersioned(), null, null, timeoutMs);
            } catch (VoldemortApplicationException e) {
                if (logger.isDebugEnabled())
                    logger.debug("Read repair cancelled due to application level exception on node " + v.getNodeId() + " for key '" + ByteUtils.toHexString(v.getKey().get()) + "' with version " + v.getVersion() + ": " + e.getMessage());
            } catch (Exception e) {
                logger.debug("Read repair failed: ", e);
            }
        }
        if (logger.isDebugEnabled()) {
            String logStr = "Repaired (node, key, version): (";
            for (NodeValue<ByteArray, byte[]> v : toReadRepair) {
                logStr += "(" + v.getNodeId() + ", " + v.getKey() + "," + v.getVersion() + ") ";
            }
            logStr += "in " + (System.nanoTime() - startTimeNs) + " ns";
            logger.debug(logStr);
        }
    }
    pipeline.addEvent(completeEvent);
}
Also used : NodeValue(voldemort.store.routed.NodeValue) NonblockingStore(voldemort.store.nonblockingstore.NonblockingStore) VoldemortApplicationException(voldemort.VoldemortApplicationException) ByteArray(voldemort.utils.ByteArray) VoldemortApplicationException(voldemort.VoldemortApplicationException)

Aggregations

ByteArray (voldemort.utils.ByteArray)309 Versioned (voldemort.versioning.Versioned)130 Test (org.junit.Test)125 VoldemortException (voldemort.VoldemortException)67 VectorClock (voldemort.versioning.VectorClock)65 ArrayList (java.util.ArrayList)61 Node (voldemort.cluster.Node)61 List (java.util.List)58 HashMap (java.util.HashMap)53 StoreDefinition (voldemort.store.StoreDefinition)49 Cluster (voldemort.cluster.Cluster)33 AbstractByteArrayStoreTest (voldemort.store.AbstractByteArrayStoreTest)31 Store (voldemort.store.Store)31 ObsoleteVersionException (voldemort.versioning.ObsoleteVersionException)31 IOException (java.io.IOException)30 Slop (voldemort.store.slop.Slop)29 Map (java.util.Map)28 Pair (voldemort.utils.Pair)28 UnreachableStoreException (voldemort.store.UnreachableStoreException)26 StatTrackingStore (voldemort.store.stats.StatTrackingStore)25