Search in sources :

Example 46 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class BdbRevertPidScanToBase method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    VersionedSerializer<byte[]> versionedSerializer = new VersionedSerializer<byte[]>(new IdentitySerializer());
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        // pull out the real key
        byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
        for (Versioned<byte[]> val : vals) {
            OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), new DatabaseEntry(versionedSerializer.toBytes(val)));
            if (OperationStatus.SUCCESS != putStatus) {
                String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(stripedKey);
                logger.error(errorStr);
                throw new Exception(errorStr);
            }
            scanCount++;
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
Also used : Versioned(voldemort.versioning.Versioned) DatabaseEntry(com.sleepycat.je.DatabaseEntry) VersionedSerializer(voldemort.serialization.VersionedSerializer) OperationStatus(com.sleepycat.je.OperationStatus) IdentitySerializer(voldemort.serialization.IdentitySerializer)

Example 47 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class BdbRevertPidScanToNewDup method transfer.

@Override
public void transfer() throws Exception {
    cursor = srcDB.openCursor(null, null);
    DatabaseEntry keyEntry = new DatabaseEntry();
    DatabaseEntry valueEntry = new DatabaseEntry();
    List<Versioned<byte[]>> vals;
    long startTime = System.currentTimeMillis();
    int scanCount = 0;
    int keyCount = 0;
    while (cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
        keyCount++;
        vals = StoreBinaryFormat.fromByteArray(valueEntry.getData());
        scanCount += vals.size();
        // pull out the real key
        byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData());
        OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), valueEntry);
        if (OperationStatus.SUCCESS != putStatus) {
            String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData());
            logger.error(errorStr);
            throw new Exception(errorStr);
        }
        if (scanCount % 1000000 == 0)
            logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
    }
    logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs");
}
Also used : Versioned(voldemort.versioning.Versioned) OperationStatus(com.sleepycat.je.OperationStatus) DatabaseEntry(com.sleepycat.je.DatabaseEntry)

Example 48 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class VersionedPutPruneJob method operate.

@Override
public void operate() throws Exception {
    StoreDefinition storeDef = StoreDefinitionUtils.getStoreDefinitionWithName(metadataStore.getStoreDefList(), storeName);
    if (storeDef == null) {
        throw new VoldemortException("Unknown store " + storeName);
    }
    if (isWritableStore(storeDef)) {
        // Lets generate routing strategy for this storage engine
        StoreRoutingPlan routingPlan = new StoreRoutingPlan(metadataStore.getCluster(), storeDef);
        logger.info("Pruning store " + storeDef.getName());
        StorageEngine<ByteArray, byte[], byte[]> engine = storeRepo.getStorageEngine(storeDef.getName());
        iterator = engine.keys();
        long itemsScanned = 0;
        long numPrunedKeys = 0;
        while (iterator.hasNext()) {
            ByteArray key = iterator.next();
            KeyLockHandle<byte[]> lockHandle = null;
            try {
                lockHandle = engine.getAndLock(key);
                List<Versioned<byte[]>> vals = lockHandle.getValues();
                List<Integer> keyReplicas = routingPlan.getReplicationNodeList(routingPlan.getMasterPartitionId(key.get()));
                MutableBoolean didPrune = new MutableBoolean(false);
                List<Versioned<byte[]>> prunedVals = pruneNonReplicaEntries(vals, keyReplicas, didPrune);
                // happened. Optimization to reduce load on storage
                if (didPrune.booleanValue()) {
                    List<Versioned<byte[]>> resolvedVals = VectorClockUtils.resolveVersions(prunedVals);
                    // TODO this is only implemented for BDB for now
                    lockHandle.setValues(resolvedVals);
                    engine.putAndUnlock(key, lockHandle);
                    numPrunedKeys = this.numKeysUpdatedThisRun.incrementAndGet();
                } else {
                    // if we did not prune, still need to let go of the lock
                    engine.releaseLock(lockHandle);
                }
                itemsScanned = this.numKeysScannedThisRun.incrementAndGet();
                throttler.maybeThrottle(1);
                if (itemsScanned % STAT_RECORDS_INTERVAL == 0) {
                    logger.info("#Scanned:" + itemsScanned + " #Pruned:" + numPrunedKeys);
                }
            } catch (Exception e) {
                throw e;
            } finally {
                if (lockHandle != null && !lockHandle.isClosed()) {
                    engine.releaseLock(lockHandle);
                }
            }
        }
        logger.info("Completed store " + storeDef.getName() + " #Scanned:" + itemsScanned + " #Pruned:" + numPrunedKeys);
    }
}
Also used : StoreRoutingPlan(voldemort.routing.StoreRoutingPlan) Versioned(voldemort.versioning.Versioned) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) VoldemortException(voldemort.VoldemortException) VoldemortException(voldemort.VoldemortException) StoreDefinition(voldemort.store.StoreDefinition) ByteArray(voldemort.utils.ByteArray)

Example 49 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class AbstractNonZonedRebalanceTest method testServerSideRouting.

@Test(timeout = 600000)
public void testServerSideRouting() throws Exception {
    logger.info("Starting testServerSideRouting");
    try {
        final Cluster currentCluster = ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3, 4, 5, 6 }, { 7, 8 } });
        final Cluster finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 1, Lists.newArrayList(2, 3));
        final List<Integer> serverList = Arrays.asList(0, 1);
        Map<String, String> configProps = new HashMap<String, String>();
        configProps.put("admin.max.threads", "50");
        configProps.put("enable.server.routing", "true");
        final Cluster updatedCurrentCluster = startServers(currentCluster, storeDefFileWithReplication, serverList, configProps);
        ExecutorService executors = Executors.newFixedThreadPool(2);
        final AtomicBoolean rebalancingToken = new AtomicBoolean(false);
        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
        String bootstrapUrl = getBootstrapUrl(currentCluster, 0);
        int maxParallel = 2;
        final ClusterTestUtils.RebalanceKit rebalanceKit = ClusterTestUtils.getRebalanceKit(bootstrapUrl, maxParallel, finalCluster);
        // Populate the two stores
        populateData(updatedCurrentCluster, roStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), true);
        populateData(updatedCurrentCluster, rwStoreDefWithReplication, rebalanceKit.controller.getAdminClient(), false);
        Node node = updatedCurrentCluster.getNodeById(1);
        final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRW = getSocketStore(testStoreNameRW, node.getHost(), node.getSocketPort(), true);
        final Store<ByteArray, byte[], byte[]> serverSideRoutingStoreRO = getSocketStore(testStoreNameRO, node.getHost(), node.getSocketPort(), true);
        final CountDownLatch latch = new CountDownLatch(1);
        // start get operation.
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    List<String> keys = new ArrayList<String>(testEntries.keySet());
                    while (!rebalancingToken.get()) {
                        // should always able to get values.
                        int index = (int) (Math.random() * keys.size());
                        // should get a valid value
                        try {
                            List<Versioned<byte[]>> values = serverSideRoutingStoreRW.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
                            assertEquals("serverSideRoutingStore should return value.", 1, values.size());
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
                            values = serverSideRoutingStoreRO.get(new ByteArray(ByteUtils.getBytes(keys.get(index), "UTF-8")), null);
                            assertEquals("serverSideRoutingStore should return value.", 1, values.size());
                            assertEquals("Value returned should be good", new Versioned<String>(testEntries.get(keys.get(index))), new Versioned<String>(ByteUtils.getString(values.get(0).getValue(), "UTF-8"), values.get(0).getVersion()));
                        } catch (UnreachableStoreException e) {
                        // ignore
                        } catch (Exception e) {
                            exceptions.add(e);
                        }
                    }
                    latch.countDown();
                } catch (Exception e) {
                    exceptions.add(e);
                }
            }
        });
        executors.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    Thread.sleep(500);
                    rebalanceAndCheck(rebalanceKit.plan, rebalanceKit.controller, Arrays.asList(0, 1));
                    Thread.sleep(500);
                    rebalancingToken.set(true);
                    checkConsistentMetadata(finalCluster, serverList);
                } catch (Exception e) {
                    exceptions.add(e);
                } finally {
                    // loop.
                    try {
                        latch.await(300, TimeUnit.SECONDS);
                        stopServer(serverList);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        });
        executors.shutdown();
        executors.awaitTermination(300, TimeUnit.SECONDS);
        // check No Exception
        if (exceptions.size() > 0) {
            for (Exception e : exceptions) {
                e.printStackTrace();
            }
            fail("Should not see any exceptions !!");
        }
    } catch (AssertionError ae) {
        logger.error("Assertion broken in testServerSideRouting ", ae);
        throw ae;
    }
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) Node(voldemort.cluster.Node) ByteArray(voldemort.utils.ByteArray) List(java.util.List) ArrayList(java.util.ArrayList) UnreachableStoreException(voldemort.store.UnreachableStoreException) Cluster(voldemort.cluster.Cluster) CountDownLatch(java.util.concurrent.CountDownLatch) UnreachableStoreException(voldemort.store.UnreachableStoreException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) IOException(java.io.IOException) InvalidMetadataException(voldemort.store.InvalidMetadataException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ClusterTestUtils(voldemort.ClusterTestUtils) ExecutorService(java.util.concurrent.ExecutorService) Test(org.junit.Test)

Example 50 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class AbstractRebalanceTest method checkForKeyExistence.

/**
     * REFACTOR: these should belong AdminClient so existence checks can be done
     * easily across the board
     * 
     * @param admin
     * @param serverId
     * @param store
     * @param keyList
     */
protected void checkForKeyExistence(AdminClient admin, int serverId, String store, List<ByteArray> keyList) {
    // do the positive tests
    Iterator<QueryKeyResult> positiveTestResultsItr = admin.streamingOps.queryKeys(serverId, store, keyList.iterator());
    while (positiveTestResultsItr.hasNext()) {
        QueryKeyResult item = positiveTestResultsItr.next();
        ByteArray key = item.getKey();
        List<Versioned<byte[]>> vals = item.getValues();
        Exception e = item.getException();
        assertEquals("Error fetching key " + key, null, e);
        assertEquals("Value not found for key " + key, true, vals != null & vals.size() != 0);
    }
}
Also used : Versioned(voldemort.versioning.Versioned) ByteArray(voldemort.utils.ByteArray) VoldemortException(voldemort.VoldemortException) QueryKeyResult(voldemort.client.protocol.admin.QueryKeyResult)

Aggregations

Versioned (voldemort.versioning.Versioned)214 ByteArray (voldemort.utils.ByteArray)130 Test (org.junit.Test)88 VectorClock (voldemort.versioning.VectorClock)73 ArrayList (java.util.ArrayList)56 VoldemortException (voldemort.VoldemortException)48 List (java.util.List)36 StoreDefinition (voldemort.store.StoreDefinition)29 HashMap (java.util.HashMap)28 IOException (java.io.IOException)27 Node (voldemort.cluster.Node)27 Pair (voldemort.utils.Pair)25 Slop (voldemort.store.slop.Slop)22 ObsoleteVersionException (voldemort.versioning.ObsoleteVersionException)22 Map (java.util.Map)21 Cluster (voldemort.cluster.Cluster)17 AbstractByteArrayStoreTest (voldemort.store.AbstractByteArrayStoreTest)15 InvalidMetadataException (voldemort.store.InvalidMetadataException)13 PersistenceFailureException (voldemort.store.PersistenceFailureException)13 Version (voldemort.versioning.Version)13