Examples with Versioned voldemort.versioning.Versioned used on opensource projects

Example 46 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class BdbRevertPidScanToBase method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    VersionedSerializer<byte[]> versionedSerializer = new VersionedSerializer<byte[]>(new IdentitySerializer());
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        // pull out the real key
        byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
        for (Versioned<byte[]> val : vals) {
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), new DatabaseEntry(versionedSerializer.toBytes(val)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(stripedKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            scanCount++;
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}

Example 47 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class BdbRevertPidScanToNewDup method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        scanCount += vals.size();
        // pull out the real key
        byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), valueEntry);
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData());
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}

Example 48 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class VersionedPutPruneJob method operate.

@Override
public void operate() throws Exception {
    StoreDefinition storeDef = StoreDefinitionUtils.getStoreDefinitionWithName(metadataStore.getStoreDefList(), storeName);
    if (storeDef == null) {
        throw new VoldemortException("Unknown store " + storeName);
    }
    if (isWritableStore(storeDef)) {
        // Lets generate routing strategy for this storage engine
        StoreRoutingPlan routingPlan = new StoreRoutingPlan(metadataStore.getCluster(), storeDef);
        logger.info("Pruning store " + storeDef.getName());
        StorageEngine<ByteArray, byte[], byte[]> engine = storeRepo.getStorageEngine(storeDef.getName());
        iterator = engine.keys();
        long itemsScanned = 0;
        long numPrunedKeys = 0;
        while (iterator.hasNext()) {
            ByteArray key = iterator.next();
            KeyLockHandle<byte[]> lockHandle = null;
            try {
                lockHandle = engine.getAndLock(key);
                List<Versioned<byte[]>> vals = lockHandle.getValues();
                List<Integer> keyReplicas = routingPlan.getReplicationNodeList(routingPlan.getMasterPartitionId(key.get()));
                MutableBoolean didPrune = new MutableBoolean(false);
                List<Versioned<byte[]>> prunedVals = pruneNonReplicaEntries(vals, keyReplicas, didPrune);
                // happened. Optimization to reduce load on storage
                if (didPrune.booleanValue()) {
                    List<Versioned<byte[]>> resolvedVals = VectorClockUtils.resolveVersions(prunedVals);
                    // TODO this is only implemented for BDB for now
                    lockHandle.setValues(resolvedVals);
                    engine.putAndUnlock(key, lockHandle);
                    numPrunedKeys = this.numKeysUpdatedThisRun.incrementAndGet();
                } else {
                    // if we did not prune, still need to let go of the lock
                    engine.releaseLock(lockHandle);
                }
                itemsScanned = this.numKeysScannedThisRun.incrementAndGet();
                throttler.maybeThrottle(1);
                if (itemsScanned % STAT_RECORDS_INTERVAL == 0) {
                    logger.info("#Scanned:" + itemsScanned + " #Pruned:" + numPrunedKeys);
                }
            } catch (Exception e) {
                throw e;
            } finally {
                if (lockHandle != null && !lockHandle.isClosed()) {
                    engine.releaseLock(lockHandle);
                }
            }
        }
        logger.info("Completed store " + storeDef.getName() + " #Scanned:" + itemsScanned + " #Pruned:" + numPrunedKeys);
    }
}

Example 49 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testServerSideRouting.

@Test(timeout = 600000)
public void testServerSideRouting() throws Exception {
    logger.info("Starting testServerSideRouting");
    try {
        final Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6 }, { 7, 8 } });
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
        final List<Integer> serverList = Arrays.asList(0, 1);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "50");
        configProps.put("enable.server.routing", "true");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingToken = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        // Populate the two stores
        populateData(updatedCurrentCluster, roStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), true);
        populateData(updatedCurrentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
        Node node = updatedCurrentCluster.getNodeById(1);
        final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRW = getSocketStore(testStoreNameRW, node.getHost(), node.getSocketPort(), true);
        final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRO = getSocketStore(testStoreNameRO, node.getHost(), node.getSocketPort(), true);
        final CountDownLatch latch = new CountDownLatch(1);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    List<String> keys = new ArrayList<String>(testEntries.keySet());
                    while (!rebalancingToken.get()) {
                        // should always able to get values.
                        int index = (int) (Math.random() * keys.size());
                        // should get a valid value
                        try {
                            List<Versioned<byte[]>> values = serverSideRoutingStoreRW.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
                            assertEquals("serverSideRoutingStore should return value.", 1, values.size());
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
                            values = serverSideRoutingStoreRO.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
                            assertEquals("serverSideRoutingStore should return value.", 1, values.size());
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
                        } catch (UnreachableStoreException e) {
                        // ignore
                        } catch (Exception e) {
                            exceptions.add(e);
                        }
                    }
                    latch.countDown();
                } catch (Exception e) {
                    exceptions.add(e);
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    Thread.sleep(500);
                    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1));
                    Thread.sleep(500);
                    rebalancingToken.set(true);
                    checkConsistentMetadata(finalCluster, serverList);
                } catch (Exception e) {
                    exceptions.add(e);
                } finally {
                    // loop.
                    try {
                        latch.await(300, TimeUnit.SECONDS);
                        stopServer(serverList);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        });
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions !!");
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testServerSideRouting ", ae);
        throw ae;
    }
}

Example 50 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class AbstractRebalanceTest method checkForKeyExistence.

/**
     * REFACTOR: these should belong AdminClient so existence checks can be done
     * easily across the board
     * 
     * @param admin
     * @param serverId
     * @param store
     * @param keyList
     */
protected void checkForKeyExistence(AdminClient admin, int serverId, String store, List<ByteArray> keyList) {
    // do the positive tests
    Iterator<QueryKeyResult> positiveTestResultsItr = admin.streamingOps.queryKeys(serverId, store, keyList.iterator());
    while (positiveTestResultsItr.hasNext()) {
        QueryKeyResult item = positiveTestResultsItr.next();
        ByteArray key = item.getKey();
        List<Versioned<byte[]>> vals = item.getValues();
        Exception e = item.getException();
        assertEquals("Error fetching key " + key, null, e);
        assertEquals("Value not found for key " + key, true, vals != null & vals.size() != 0);
    }
}