Examples with ObsoleteVersionException voldemort.versioning.ObsoleteVersionException used on opensource projects

Example 11 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class BlockingSlopPusherJob method run.

/**
     * Loop over entries in the slop table and attempt to push them to the
     * deserving server
     */
public void run() {
    // don't try to run slop pusher job when rebalancing
    if (metadataStore.getServerStateUnlocked().equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
        logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
        return;
    }
    logger.info("Started blocking slop pusher job at " + new Date());
    Cluster cluster = metadataStore.getCluster();
    failureDetector.getConfig().setCluster(cluster);
    Set<String> storeNames = StoreDefinitionUtils.getStoreNamesSet(metadataStore.getStoreDefList());
    ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
    Map<Integer, Long> attemptedByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    Map<Integer, Long> succeededByNode = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
    long slopsPushed = 0L;
    long attemptedPushes = 0L;
    for (Node node : cluster.getNodes()) {
        attemptedByNode.put(node.getId(), 0L);
        succeededByNode.put(node.getId(), 0L);
    }
    acquireRepairPermit();
    try {
        SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
        StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
        EventThrottler throttler = new EventThrottler(maxWriteBytesPerSec);
        iterator = slopStore.entries();
        while (iterator.hasNext()) {
            if (Thread.interrupted())
                throw new InterruptedException("Slop pusher job cancelled");
            try {
                Pair<ByteArray, Versioned<Slop>> keyAndVal;
                try {
                    keyAndVal = iterator.next();
                } catch (Exception e) {
                    logger.error("Exception in iterator, escaping the loop ", e);
                    break;
                }
                Versioned<Slop> versioned = keyAndVal.getSecond();
                Slop slop = versioned.getValue();
                int nodeId = slop.getNodeId();
                // check for dead slops
                if (isSlopDead(cluster, storeNames, versioned.getValue())) {
                    handleDeadSlop(slopStorageEngine, keyAndVal);
                    // the next slop.
                    continue;
                }
                Node node = cluster.getNodeById(nodeId);
                attemptedPushes++;
                if (attemptedPushes % 10000 == 0) {
                    logger.info("Attempted pushing " + attemptedPushes + " slops");
                }
                Long attempted = attemptedByNode.get(nodeId);
                attemptedByNode.put(nodeId, attempted + 1L);
                if (failureDetector.isAvailable(node)) {
                    Store<ByteArray, byte[], byte[]> store = storeRepo.getNodeStore(slop.getStoreName(), node.getId());
                    Long startNs = System.nanoTime();
                    int nBytes = 0;
                    try {
                        nBytes = slop.getKey().length();
                        if (slop.getOperation() == Operation.PUT) {
                            store.put(slop.getKey(), new Versioned<byte[]>(slop.getValue(), versioned.getVersion()), slop.getTransforms());
                            nBytes += slop.getValue().length + ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                        } else if (slop.getOperation() == Operation.DELETE) {
                            nBytes += ((VectorClock) versioned.getVersion()).sizeInBytes() + 1;
                            store.delete(slop.getKey(), versioned.getVersion());
                        } else {
                            logger.error("Unknown slop operation: " + slop.getOperation());
                            continue;
                        }
                        failureDetector.recordSuccess(node, deltaMs(startNs));
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (ObsoleteVersionException e) {
                        // okay it is old, just delete it
                        slopStore.delete(slop.makeKey(), versioned.getVersion());
                        slopsPushed++;
                        // Increment succeeded
                        Long succeeded = succeededByNode.get(nodeId);
                        succeededByNode.put(nodeId, succeeded + 1L);
                        // Throttle the bytes...
                        throttler.maybeThrottle(nBytes);
                    } catch (UnreachableStoreException e) {
                        failureDetector.recordException(node, deltaMs(startNs), e);
                    }
                }
            } catch (Exception e) {
                logger.error(e, e);
            }
        }
        // Only if we reached here do we update stats
        logger.log(attemptedPushes > 0 ? Level.INFO : Level.DEBUG, "Attempted " + attemptedPushes + " hinted handoff pushes of which " + slopsPushed + " succeeded.");
        Map<Integer, Long> outstanding = Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
        for (int nodeId : succeededByNode.keySet()) {
            outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
        }
        slopStorageEngine.resetStats(outstanding);
    } catch (Exception e) {
        logger.error(e, e);
    } finally {
        try {
            if (iterator != null)
                iterator.close();
        } catch (Exception e) {
            logger.error("Failed to close iterator.", e);
        }
        this.repairPermits.release(this.getClass().getCanonicalName());
    }
}

Example 12 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class VoldemortNativeRequestHandler method handleRequest.

@Override
public StreamRequestHandler handleRequest(final DataInputStream inputStream, final DataOutputStream outputStream, final ByteBufferContainer outputContainer) throws IOException {
    long startTimeMs = -1;
    long startTimeNs = -1;
    if (logger.isDebugEnabled()) {
        startTimeMs = System.currentTimeMillis();
        startTimeNs = System.nanoTime();
    }
    byte opCode = inputStream.readByte();
    String storeName = inputStream.readUTF();
    RequestRoutingType routingType = getRoutingType(inputStream);
    Store<ByteArray, byte[], byte[]> store = getStore(storeName, routingType);
    if (store == null) {
        clearBuffer(outputContainer);
        writeException(outputStream, new VoldemortException("No store named: '" + storeName + "'."));
        return null;
    }
    ClientRequestHandler requestHandler = getClientRequestHandler(opCode, store);
    try {
        requestHandler.parseRequest(inputStream);
        requestHandler.processRequest();
    } catch (VoldemortException e) {
        // they are harmless and indicates normal mode of operation.
        if (!(e instanceof ObsoleteVersionException)) {
            logger.error("Store: " + storeName + ". Error: " + e.getMessage());
        }
        clearBuffer(outputContainer);
        writeException(outputStream, e);
        return null;
    }
    // We are done with Input, clear the buffers
    clearBuffer(outputContainer);
    int size = requestHandler.getResponseSize();
    if (outputContainer != null) {
        outputContainer.growBuffer(size);
    }
    requestHandler.writeResponse(outputStream);
    outputStream.flush();
    if (logger.isDebugEnabled()) {
        String debugPrefix = "OpCode: " + opCode + ", started at: " + startTimeMs + ", handlerRef: " + System.identityHashCode(inputStream) + ", Elapsed : " + (System.nanoTime() - startTimeNs) + " ns, ";
        logger.debug(debugPrefix + requestHandler.getDebugMessage());
    }
    return null;
}

Example 13 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class UpdateSlopEntriesRequestHandler method handleRequest.

public StreamRequestHandlerState handleRequest(DataInputStream inputStream, DataOutputStream outputStream) throws IOException {
    if (!metadataStore.getSlopStreamingEnabledUnlocked()) {
        throw new SlopStreamingDisabledException("Slop streaming is disabled on node " + metadataStore.getNodeId() + " under " + metadataStore.getServerStateUnlocked() + " state.");
    }
    long startNs = System.nanoTime();
    if (request == null) {
        int size = 0;
        try {
            size = inputStream.readInt();
        } catch (EOFException e) {
            if (logger.isTraceEnabled())
                logger.trace("Incomplete read for message size");
            networkTimeNs += System.nanoTime() - startNs;
            return StreamRequestHandlerState.INCOMPLETE_READ;
        }
        if (size == -1) {
            if (logger.isTraceEnabled())
                logger.trace("Message size -1, completed slop update");
            return StreamRequestHandlerState.COMPLETE;
        }
        if (logger.isTraceEnabled())
            logger.trace("UpdateSlopEntriesRequest message size: " + size);
        byte[] input = new byte[size];
        try {
            ByteUtils.read(inputStream, input);
            networkTimeNs += Utils.elapsedTimeNs(startNs, System.nanoTime());
        } catch (EOFException e) {
            if (logger.isTraceEnabled())
                logger.trace("Incomplete read for message");
            return StreamRequestHandlerState.INCOMPLETE_READ;
        }
        VAdminProto.UpdateSlopEntriesRequest.Builder builder = VAdminProto.UpdateSlopEntriesRequest.newBuilder();
        builder.mergeFrom(input);
        request = builder.build();
    }
    StorageEngine<ByteArray, byte[], byte[]> storageEngine = AdminServiceRequestHandler.getStorageEngine(storeRepository, request.getStore());
    StreamingStats streamStats = null;
    if (isJmxEnabled) {
        streamStats = storeRepository.getStreamingStats(storageEngine.getName());
        streamStats.reportNetworkTime(Operation.SLOP_UPDATE, networkTimeNs);
    }
    networkTimeNs = 0;
    ByteArray key = ProtoUtils.decodeBytes(request.getKey());
    VectorClock vectorClock = ProtoUtils.decodeClock(request.getVersion());
    switch(request.getRequestType()) {
        case PUT:
            try {
                // Retrieve the transform if its exists
                byte[] transforms = null;
                if (request.hasTransform()) {
                    transforms = ProtoUtils.decodeBytes(request.getTransform()).get();
                }
                // Retrieve the value
                byte[] value = ProtoUtils.decodeBytes(request.getValue()).get();
                startNs = System.nanoTime();
                storageEngine.put(key, Versioned.value(value, vectorClock), transforms);
                if (isJmxEnabled)
                    streamStats.reportStorageTime(Operation.SLOP_UPDATE, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                if (logger.isTraceEnabled())
                    logger.trace("updateSlopEntries (Streaming put) successful on key:" + key + " of store: " + request.getStore());
            } catch (ObsoleteVersionException e) {
                // log and ignore
                if (logger.isDebugEnabled())
                    logger.debug("updateSlopEntries (Streaming put) threw ObsoleteVersionException, Ignoring.");
            }
            break;
        case DELETE:
            try {
                startNs = System.nanoTime();
                storageEngine.delete(key, vectorClock);
                if (isJmxEnabled)
                    streamStats.reportStorageTime(Operation.SLOP_UPDATE, System.nanoTime() - startNs);
                if (logger.isTraceEnabled())
                    logger.trace("updateSlopEntries (Streaming delete) successful");
            } catch (ObsoleteVersionException e) {
                // log and ignore
                if (logger.isDebugEnabled())
                    logger.debug("updateSlopEntries (Streaming delete) threw ObsoleteVersionException, Ignoring.");
            }
            break;
        default:
            throw new VoldemortException("Unsupported operation ");
    }
    // log progress
    counter++;
    if (isJmxEnabled)
        streamStats.reportStreamingPut(Operation.SLOP_UPDATE);
    if (0 == counter % 100000) {
        long totalTime = (System.currentTimeMillis() - startTime) / 1000;
        if (logger.isDebugEnabled())
            logger.debug("updateSlopEntries() updated " + counter + " entries in " + totalTime + " s");
    }
    request = null;
    return StreamRequestHandlerState.READING;
}

Example 14 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method populateData.

protected void populateData(Cluster cluster, StoreDefinition storeDef, AdminClient adminClient, boolean isReadOnly) throws Exception {
    // Populate Read write stores
    if (!isReadOnly) {
        // Create SocketStores for each Node first
        Map<Integer, Store<ByteArray, byte[], byte[]>> storeMap = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
        for (Node node : cluster.getNodes()) {
            storeMap.put(node.getId(), getSocketStore(storeDef.getName(), node.getHost(), node.getSocketPort()));
        }
        BaseStoreRoutingPlan storeInstance = new BaseStoreRoutingPlan(cluster, storeDef);
        for (Entry<String, String> entry : testEntries.entrySet()) {
            ByteArray keyBytes = new ByteArray(ByteUtils.getBytes(entry.getKey(), "UTF-8"));
            List<Integer> preferenceNodes = storeInstance.getReplicationNodeList(keyBytes.get());
            // Go over every node
            for (int nodeId : preferenceNodes) {
                try {
                    storeMap.get(nodeId).put(keyBytes, new Versioned<byte[]>(ByteUtils.getBytes(entry.getValue(), "UTF-8")), null);
                } catch (ObsoleteVersionException e) {
                    logger.info("Why are we seeing this at all here ?? ");
                    e.printStackTrace();
                }
            }
        }
        // close all socket stores
        for (Store<ByteArray, byte[], byte[]> store : storeMap.values()) {
            store.close();
        }
    } else {
        // Populate Read only stores
        File baseDir = TestUtils.createTempDir();
        JsonReader reader = ReadOnlyStorageEngineTestInstance.makeTestDataReader(testEntries, baseDir);
        RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
        File outputDir = TestUtils.createTempDir(baseDir);
        JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testEntries.size() / 5, 1, NUM_RO_CHUNKS_PER_BUCKET, 10000, false);
        storeBuilder.build(ReadOnlyStorageFormat.READONLY_V2);
        AdminStoreSwapper swapper = new AdminStoreSwapper(Executors.newFixedThreadPool(cluster.getNumberOfNodes()), adminClient, 100000);
        swapper.fetchAndSwapStoreData(testStoreNameRO, outputDir.getAbsolutePath(), 1L);
    }
}

Example 15 with ObsoleteVersionException

use of voldemort.versioning.ObsoleteVersionException in project voldemort by voldemort.

the class AbstractZonedRebalanceTest method populateData.

private void populateData(Cluster cluster, StoreDefinition storeDef) throws Exception {
    // Create SocketStores for each Node first
    Map<Integer, Store<ByteArray, byte[], byte[]>> storeMap = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        storeMap.put(node.getId(), getSocketStore(storeDef.getName(), node.getHost(), node.getSocketPort()));
    }
    BaseStoreRoutingPlan storeInstance = new BaseStoreRoutingPlan(cluster, storeDef);
    for (Entry<String, String> entry : testEntries.entrySet()) {
        ByteArray keyBytes = new ByteArray(ByteUtils.getBytes(entry.getKey(), "UTF-8"));
        List<Integer> preferenceNodes = storeInstance.getReplicationNodeList(keyBytes.get());
        // Go over every node
        for (int nodeId : preferenceNodes) {
            try {
                storeMap.get(nodeId).put(keyBytes, new Versioned<byte[]>(ByteUtils.getBytes(entry.getValue(), "UTF-8")), null);
            } catch (ObsoleteVersionException e) {
                logger.info("Why are we seeing this at all here ?? ");
                e.printStackTrace();
            }
        }
    }
    // close all socket stores
    for (Store<ByteArray, byte[], byte[]> store : storeMap.values()) {
        store.close();
    }
}