Examples with Versioned voldemort.versioning.Versioned used on opensource projects

Example 36 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class AddNodeAction method update.

/**
     * @throws ObsoleteVersionException if a concurrent modification (remove or
     *         another add) has not completed modifying the structure.
     * @throws ArrayIndexOutOfBoundsException if no more ids left to identify
     *         object
     */
@Override
public void update(StoreClient<Map<String, Object>, Map<String, Object>> storeClient) {
    _storeClient = storeClient;
    VListKey<K> newKey = new VListKey<K>(_key, 0);
    Versioned<Map<String, Object>> firstNodeMap = storeClient.get(newKey.mapValue());
    // adding first node of the list
    if (firstNodeMap == null) {
        Versioned<Map<String, Object>> newNode = new Versioned<Map<String, Object>>((new VListNode<E>(_value, 0, VStack.NULL_ID, VStack.NULL_ID, true)).mapValue());
        // throws ObsoleteVersionException if another process has created a
        // new node already
        storeClient.put(newKey.mapValue(), newNode);
    } else // add to front of list
    {
        Versioned<VListNode<E>> firstNode = new Versioned<VListNode<E>>(VListNode.<E>valueOf(firstNodeMap.getValue()), firstNodeMap.getVersion());
        if (!firstNode.getValue().isStable()) {
            throw new ObsoleteVersionException("cannot add when list node is not stable");
        }
        // set stable flag to false
        Map<String, Object> tmpMap = new HashMap<String, Object>(firstNodeMap.getValue());
        tmpMap.put(VListNode.STABLE, false);
        storeClient.put(newKey.mapValue(), new Versioned<Map<String, Object>>(tmpMap, firstNodeMap.getVersion()));
        _rollback.put(0, firstNodeMap.getValue());
        int newId;
        int nextId = firstNode.getValue().getNextId();
        newId = (nextId == VStack.NULL_ID) ? 1 : nextId + 1;
        if (newId == Integer.MAX_VALUE)
            throw new ArrayIndexOutOfBoundsException(newId + " out of bounds");
        Versioned<VListNode<E>> nextNode = null;
        VListKey<K> nextKey = new VListKey<K>(_key, nextId);
        if (nextId != VStack.NULL_ID) {
            Versioned<Map<String, Object>> nextNodeMap = storeClient.get(nextKey.mapValue());
            if (nextNodeMap == null)
                throw new ObsoleteVersionException("possible concurrent modification");
            nextNode = new Versioned<VListNode<E>>(VListNode.<E>valueOf(nextNodeMap.getValue()), nextNodeMap.getVersion());
            if (!nextNode.getValue().isStable()) {
                throw new ObsoleteVersionException("cannot add when list node is not stable");
            }
            // set stable flag to false
            tmpMap = new HashMap<String, Object>(nextNode.getValue().mapValue());
            tmpMap.put(VListNode.STABLE, false);
            storeClient.put(nextKey.mapValue(), new Versioned<Map<String, Object>>(tmpMap, nextNode.getVersion()));
            _rollback.put(nextId, nextNode.getValue().mapValue());
        }
        // insert new node
        Map<String, Object> newNode = (new VListNode<E>(_value, 0, VStack.NULL_ID, newId, true)).mapValue();
        // don't need to specify versioned because node is already "locked"
        storeClient.put(newKey.mapValue(), newNode);
        // move first node to next index
        VListKey<K> firstKey = new VListKey<K>(_key, newId);
        firstNode = new Versioned<VListNode<E>>(new VListNode<E>(firstNode.getValue().getValue(), newId, 0, firstNode.getValue().getNextId(), true));
        // don't need to specify versioned because node is already "locked"
        storeClient.put(firstKey.mapValue(), firstNode.getValue().mapValue());
        // redefine previous pointer on next node
        if (nextNode != null) {
            if (!storeClient.applyUpdate(new UpdateNextNode<K, E>(nextNode, nextKey, newId)))
                throw new ObsoleteVersionException("unable to update node");
        }
    }
}

Example 37 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class StorageService method initializeMetadataVersions.

protected void initializeMetadataVersions(List<StoreDefinition> storeDefs) {
    Store<ByteArray, byte[], byte[]> versionStore = storeRepository.getLocalStore(SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    Properties props = new Properties();
    try {
        boolean isPropertyAdded = false;
        ByteArray metadataVersionsKey = new ByteArray(SystemStoreConstants.VERSIONS_METADATA_KEY.getBytes());
        List<Versioned<byte[]>> versionList = versionStore.get(metadataVersionsKey, null);
        VectorClock newClock = null;
        if (versionList != null && versionList.size() > 0) {
            byte[] versionsByteArray = versionList.get(0).getValue();
            if (versionsByteArray != null) {
                props.load(new ByteArrayInputStream(versionsByteArray));
            }
            newClock = (VectorClock) versionList.get(0).getVersion();
            newClock = newClock.incremented(0, System.currentTimeMillis());
        } else {
            newClock = new VectorClock();
        }
        // Check if version exists for cluster.xml
        if (!props.containsKey(SystemStoreConstants.CLUSTER_VERSION_KEY)) {
            props.setProperty(SystemStoreConstants.CLUSTER_VERSION_KEY, "0");
            isPropertyAdded = true;
        }
        // Check if version exists for stores.xml
        if (!props.containsKey(SystemStoreConstants.STORES_VERSION_KEY)) {
            props.setProperty(SystemStoreConstants.STORES_VERSION_KEY, "0");
            isPropertyAdded = true;
        }
        // Check if version exists for each store
        for (StoreDefinition def : storeDefs) {
            if (!props.containsKey(def.getName())) {
                props.setProperty(def.getName(), "0");
                isPropertyAdded = true;
            }
        }
        if (isPropertyAdded) {
            StringBuilder finalVersionList = new StringBuilder();
            for (String propName : props.stringPropertyNames()) {
                finalVersionList.append(propName + "=" + props.getProperty(propName) + "\n");
            }
            versionStore.put(metadataVersionsKey, new Versioned<byte[]>(finalVersionList.toString().getBytes(), newClock), null);
        }
    } catch (Exception e) {
        logger.error("Error while intializing metadata versions ", e);
    }
}

Example 38 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class StorageService method calculateStats.

private DataSetStats calculateStats(StorageEngine<ByteArray, byte[], byte[]> store) {
    DataSetStats stats = new DataSetStats();
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iter = store.entries();
    try {
        int count = 0;
        while (iter.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> pair = iter.next();
            VectorClock clock = (VectorClock) pair.getSecond().getVersion();
            stats.countEntry(pair.getFirst().length(), pair.getSecond().getValue().length + clock.sizeInBytes());
            if (count % 10000 == 0)
                logger.debug("Processing key " + count);
            count++;
        }
    } finally {
        iter.close();
    }
    return stats;
}

Example 39 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class BlockingSlopPusherJob method run.

/**
     * Loop over entries in the slop table and attempt to push them to the
     * deserving server
     */
public void run() {
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    logger.info("Started blocking slop pusher job at " + new Date());
    Cluster cluster = metadataStore.getCluster();
    failureDetector.getConfig().setCluster(cluster);
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    Map<Integer, Long> attemptedByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    Map<Integer, Long> succeededByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    long slopsPushed = 0L;
    long attemptedPushes = 0L;
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    acquireRepairPermit();
    try {
        SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        EventThrottler throttler = new EventThrottler(maxWriteBytesPerSec);
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            if (Thread.interrupted())
                throw new InterruptedException("Slop pusher job cancelled");
            try {
                Pair<ByteArray, Versioned<Slop>> keyAndVal;
                try {
                    keyAndVal = iterator.next();
                } catch (Exception e) {
                    logger.error("Exception in iterator, escaping the loop ", e);
                    break;
                }
                Versioned<Slop> versioned = keyAndVal.getSecond();
                Slop slop = versioned.getValue();
                int nodeId = slop.getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // the next slop.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes++;
                if (attemptedPushes % 10000 == 0) {
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                }
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (failureDetector.isAvailable(node)) {
                    Store<ByteArray, byte[], byte[]> store = storeRepo.getNodeStore(slop.getStoreName(), node.getId());
                    Long startNs = System.nanoTime();
                    int nBytes = 0;
                    try {
                        nBytes = slop.getKey().length();
                        if (slop.getOperation() == Operation.PUT) {
                            store.put(slop.getKey(), new Versioned<byte[]>(slop.getValue(), versioned.getVersion()), slop.getTransforms());
                            nBytes += slop.getValue().length + ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                        } else if (slop.getOperation() == Operation.DELETE) {
                            nBytes += ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                            store.delete(slop.getKey(), versioned.getVersion());
                        } else {
                            logger.error("Unknown slop operation: " + slop.getOperation());
                            continue;
                        }
                        failureDetector.recordSuccess(node, deltaMs(startNs));
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (ObsoleteVersionException e) {
                        // okay it is old, just delete it
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (UnreachableStoreException e) {
                        failureDetector.recordException(node, deltaMs(startNs), e);
                    }
                }
            } catch (Exception e) {
                logger.error(e, e);
            }
        }
        // Only if we reached here do we update stats
        logger.log(attemptedPushes > 0 ? Level.INFO : Level.DEBUG, "Attempted " + attemptedPushes + " hinted handoff pushes of which " + slopsPushed + " succeeded.");
        Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
        for (int nodeId : succeededByNode.keySet()) {
            outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
        }
        slopStorageEngine.resetStats(outstanding);
    } catch (Exception e) {
        logger.error(e, e);
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.error("Failed to close iterator.", e);
        }
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}

Example 40 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class AdminServiceRequestHandler method handleGetMetadata.

public VAdminProto.GetMetadataResponse handleGetMetadata(VAdminProto.GetMetadataRequest request) {
    VAdminProto.GetMetadataResponse.Builder response = VAdminProto.GetMetadataResponse.newBuilder();
    try {
        ByteArray key = ProtoUtils.decodeBytes(request.getKey());
        String keyString = ByteUtils.getString(key.get(), "UTF-8");
        if (keyString.isEmpty()) {
            // a new instance, with an empty key field.
            throw new VoldemortException("Received admin operation which got interpreted as a " + "GetMetadataRequest with an empty metadata key. This " + "typically means that the requested admin operation is not " + "supported on this version of the Voldemort server.");
        }
        List<Versioned<byte[]>> versionedList = metadataStore.get(key, null);
        if (versionedList.size() > 0) {
            Versioned<byte[]> versioned = versionedList.get(0);
            response.setVersion(ProtoUtils.encodeVersioned(versioned));
        }
    } catch (VoldemortException e) {
        response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
        String errorMessage = "handleGetMetadata failed for request(" + request.toString() + ")";
        if (e instanceof StoreNotFoundException) {
            logger.info(errorMessage + " with " + StoreNotFoundException.class.getSimpleName());
        } else {
            logger.error(errorMessage, e);
        }
    }
    return response.build();
}