Examples with VectorClock voldemort.versioning.VectorClock used on opensource projects

Example 21 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class GetVersionRequestHandler method writeResponse.

@Override
public void writeResponse(DataOutputStream outputStream) throws IOException {
    outputStream.writeShort(0);
    outputStream.writeInt(results.size());
    for (Version v : results) {
        byte[] clock = ((VectorClock) v).toBytes();
        outputStream.writeInt(clock.length);
        outputStream.write(clock);
    }
}

Example 22 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class TimeBasedUpdatePartitionEntriesStreamRequestHandler method processEntry.

@Override
protected void processEntry(ByteArray key, Versioned<byte[]> value) throws IOException {
    KeyLockHandle<byte[]> handle = null;
    try {
        handle = storageEngine.getAndLock(key);
        // determine if there is a version already with a greater ts.
        boolean foundGreaterTs = false;
        VectorClock streamedClock = (VectorClock) value.getVersion();
        for (Versioned<byte[]> versioned : handle.getValues()) {
            VectorClock storedClock = (VectorClock) versioned.getVersion();
            if (storedClock.getTimestamp() >= streamedClock.getTimestamp()) {
                foundGreaterTs = true;
                break;
            }
        }
        if (!foundGreaterTs) {
            // if what we are trying to write is the greatest version,
            // write it in and let go of lock
            List<Versioned<byte[]>> streamedVals = new ArrayList<Versioned<byte[]>>(1);
            streamedVals.add(value);
            handle.setValues(streamedVals);
            storageEngine.putAndUnlock(key, handle);
        } else {
            // back off and let go of lock, if found a version that is
            // greater than what we are trying to write in
            storageEngine.releaseLock(handle);
        }
    } catch (Exception e) {
        logger.error("Error in time based update entries", e);
        throw new IOException(e);
    } finally {
        if (handle != null && !handle.isClosed()) {
            storageEngine.releaseLock(handle);
        }
    }
}

Example 23 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class UpdateSlopEntriesRequestHandler method handleRequest.

public StreamRequestHandlerState handleRequest(DataInputStream inputStream, DataOutputStream outputStream) throws IOException {
    if (!metadataStore.getSlopStreamingEnabledUnlocked()) {
        throw new SlopStreamingDisabledException("Slop streaming is disabled on node " + metadataStore.getNodeId() + " under " + metadataStore.getServerStateUnlocked() + " state.");
    }
    long startNs = System.nanoTime();
    if (request == null) {
        int size = 0;
        try {
            size = inputStream.readInt();
        } catch (EOFException e) {
            if (logger.isTraceEnabled())
                logger.trace("Incomplete read for message size");
            networkTimeNs += System.nanoTime() - startNs;
            return StreamRequestHandlerState.INCOMPLETE_READ;
        }
        if (size == -1) {
            if (logger.isTraceEnabled())
                logger.trace("Message size -1, completed slop update");
            return StreamRequestHandlerState.COMPLETE;
        }
        if (logger.isTraceEnabled())
            logger.trace("UpdateSlopEntriesRequest message size: " + size);
        byte[] input = new byte[size];
        try {
            ByteUtils.read(inputStream, input);
            networkTimeNs += Utils.elapsedTimeNs(startNs, System.nanoTime());
        } catch (EOFException e) {
            if (logger.isTraceEnabled())
                logger.trace("Incomplete read for message");
            return StreamRequestHandlerState.INCOMPLETE_READ;
        }
        VAdminProto.UpdateSlopEntriesRequest.Builder builder = VAdminProto.UpdateSlopEntriesRequest.newBuilder();
        builder.mergeFrom(input);
        request = builder.build();
    }
    StorageEngine<ByteArray, byte[], byte[]> storageEngine = AdminServiceRequestHandler.getStorageEngine(storeRepository, request.getStore());
    StreamingStats streamStats = null;
    if (isJmxEnabled) {
        streamStats = storeRepository.getStreamingStats(storageEngine.getName());
        streamStats.reportNetworkTime(Operation.SLOP_UPDATE, networkTimeNs);
    }
    networkTimeNs = 0;
    ByteArray key = ProtoUtils.decodeBytes(request.getKey());
    VectorClock vectorClock = ProtoUtils.decodeClock(request.getVersion());
    switch(request.getRequestType()) {
        case PUT:
            try {
                // Retrieve the transform if its exists
                byte[] transforms = null;
                if (request.hasTransform()) {
                    transforms = ProtoUtils.decodeBytes(request.getTransform()).get();
                }
                // Retrieve the value
                byte[] value = ProtoUtils.decodeBytes(request.getValue()).get();
                startNs = System.nanoTime();
                storageEngine.put(key, Versioned.value(value, vectorClock), transforms);
                if (isJmxEnabled)
                    streamStats.reportStorageTime(Operation.SLOP_UPDATE, Utils.elapsedTimeNs(startNs, System.nanoTime()));
                if (logger.isTraceEnabled())
                    logger.trace("updateSlopEntries (Streaming put) successful on key:" + key + " of store: " + request.getStore());
            } catch (ObsoleteVersionException e) {
                // log and ignore
                if (logger.isDebugEnabled())
                    logger.debug("updateSlopEntries (Streaming put) threw ObsoleteVersionException, Ignoring.");
            }
            break;
        case DELETE:
            try {
                startNs = System.nanoTime();
                storageEngine.delete(key, vectorClock);
                if (isJmxEnabled)
                    streamStats.reportStorageTime(Operation.SLOP_UPDATE, System.nanoTime() - startNs);
                if (logger.isTraceEnabled())
                    logger.trace("updateSlopEntries (Streaming delete) successful");
            } catch (ObsoleteVersionException e) {
                // log and ignore
                if (logger.isDebugEnabled())
                    logger.debug("updateSlopEntries (Streaming delete) threw ObsoleteVersionException, Ignoring.");
            }
            break;
        default:
            throw new VoldemortException("Unsupported operation ");
    }
    // log progress
    counter++;
    if (isJmxEnabled)
        streamStats.reportStreamingPut(Operation.SLOP_UPDATE);
    if (0 == counter % 100000) {
        long totalTime = (System.currentTimeMillis() - startTime) / 1000;
        if (logger.isDebugEnabled())
            logger.debug("updateSlopEntries() updated " + counter + " entries in " + totalTime + " s");
    }
    request = null;
    return StreamRequestHandlerState.READING;
}

Example 24 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class BdbConvertBaseToNewDup method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    byte[] prevKey = null;
    List<Versioned<byte[]>> vals = new ArrayList<Versioned<byte[]>>();
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        scanCount++;
        if (scanCount % 1000000 == 0)
            logger.info("Converted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
        // read the value as a versioned Object
        VectorClock clock = new VectorClock(valueEntry.getData());
        byte[] bytes = ByteUtils.copy(valueEntry.getData(), clock.sizeInBytes(), valueEntry.getData().length);
        Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
        byte[] key = keyEntry.getData();
        if (prevKey != null && (ByteUtils.compare(prevKey, key) != 0)) {
            // there is a new key; write out the buffered values and
            // previous key
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(prevKey), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            vals = new ArrayList<Versioned<byte[]>>();
            keyCount++;
        }
        vals.add(value);
        prevKey = key;
    }
    if (vals.size() > 0) {
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(prevKey), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        keyCount++;
    }
    logger.info("Completed " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}

Example 25 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class BdbConvertBaseToPidScan method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    byte[] prevKey = null;
    List<Versioned<byte[]>> vals = new ArrayList<Versioned<byte[]>>();
    HashFunction hash = new FnvHashFunction();
    int totalPartitions = cluster.getNumberOfPartitions();
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        scanCount++;
        if (scanCount % 1000000 == 0)
            logger.info("Converted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
        // read the value as a versioned Object
        VectorClock clock = new VectorClock(valueEntry.getData());
        byte[] bytes = ByteUtils.copy(valueEntry.getData(), clock.sizeInBytes(), valueEntry.getData().length);
        Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
        byte[] key = keyEntry.getData();
        if (prevKey != null && (ByteUtils.compare(prevKey, key) != 0)) {
            // there is a new key; write out the buffered values and
            // previous key
            int partition = BdbConvertData.abs(hash.hash(prevKey)) % (Math.max(1, totalPartitions));
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey, partition)), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            vals = new ArrayList<Versioned<byte[]>>();
            keyCount++;
        }
        vals.add(value);
        prevKey = key;
    }
    if (vals.size() > 0) {
        int partition = BdbConvertData.abs(hash.hash(prevKey)) % (Math.max(1, totalPartitions));
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(StoreBinaryFormat.makePrefixedKey(prevKey, partition)), new DatabaseEntry(StoreBinaryFormat.toByteArray(vals)));
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(prevKey);
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        keyCount++;
    }
    logger.info("Completed " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}