Examples with Slop voldemort.store.slop.Slop used on opensource projects

Example 16 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class StreamingSlopPusherTest method testServerReplacementWithoutBounce.

@Test
public void testServerReplacementWithoutBounce() throws IOException, InterruptedException {
    startServers(0, 2);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet1 = ServerTestUtils.createRandomSlops(1, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    // Generate slops for 2
    final List<Versioned<Slop>> entrySet2 = ServerTestUtils.createRandomSlops(2, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet1, entrySet2);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(10000);
    // Now check if the slops went through and also got deleted
    Iterator<Versioned<Slop>> entryIterator = entrySet2.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(2).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    entryIterator = entrySet1.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        // did it get deleted correctly
        assertNotSame("slop should be there", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 50);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(50));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    // now replace server 1 with a new host and start it
    cluster = ServerTestUtils.updateClusterWithNewHost(cluster, 1);
    startServers(1);
    // update the meatadata store with the new cluster on node 0 and 2 (the
    // two servers that are running)
    servers[0].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);
    servers[2].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);
    // Give some time for the pusher job to figure out that server1 is up
    Thread.sleep(35000);
    // start the pusher job again
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(10000);
    // make sure the slot for server 1 is pushed to the new host
    // Now check if the slops went through and also got deleted
    entryIterator = entrySet1.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
}

Example 17 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class SlopPusherDeadSlopTest method testAutoPurge.

@Test
public void testAutoPurge() {
    try {
        // generate slops for a non existent node 2.
        List<Versioned<Slop>> deadNodeSlops = ServerTestUtils.createRandomSlops(2, 40, false, "test");
        // generate slops for a non existent store "deleted_store"
        List<Versioned<Slop>> deadStoreSlops = ServerTestUtils.createRandomSlops(0, 40, false, "deleted_store");
        // generate some valid slops and make sure they go into the
        // destination store
        List<Versioned<Slop>> validStoreSlops = ServerTestUtils.createRandomSlops(1, 40, false, "test");
        List<Versioned<Slop>> slops = new ArrayList<Versioned<Slop>>();
        slops.addAll(deadStoreSlops);
        slops.addAll(deadNodeSlops);
        slops.addAll(validStoreSlops);
        SlopSerializer slopSerializer = new SlopSerializer();
        // Populate the store with the slops
        for (Versioned<Slop> slop : slops) {
            VectorClock clock = TestUtils.getClock(1);
            NodeValue<ByteArray, byte[]> nodeValue = new NodeValue<ByteArray, byte[]>(0, slop.getValue().makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop.getValue()), clock));
            adminClient.storeOps.putNodeKeyValue("slop", nodeValue);
        }
        // wait for twice the slop interval (in case a slop push was
        // underway as we populated)
        Thread.sleep(SLOP_FREQUENCY_MS * 2);
        // Confirm the dead slops are all gone now..
        for (List<Versioned<Slop>> deadSlops : Arrays.asList(deadStoreSlops, deadNodeSlops)) {
            for (Versioned<Slop> slop : deadSlops) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("slop", 0, slop.getValue().makeKey());
                assertEquals("Slop should be purged", 0, slopEntry.size());
            }
        }
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getSingleStoreDefinitionsXml()));
        BaseStoreRoutingPlan rPlan = new BaseStoreRoutingPlan(adminClient.getAdminClientCluster(), StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, "test"));
        // Confirm the valid ones made it
        for (Versioned<Slop> slop : validStoreSlops) {
            ByteArray key = slop.getValue().getKey();
            if (rPlan.getReplicationNodeList(key.get()).contains(1)) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("test", 1, key);
                if (slop.getValue().getOperation() == Operation.DELETE) {
                    assertTrue("Delete Slop should have not reached destination", slopEntry.size() == 0);
                } else {
                    assertTrue("Put Slop should have reached destination", slopEntry.size() > 0);
                }
            }
        }
    } catch (Exception e) {
        logger.error("Test failed with", e);
        fail("unexpected exception");
    }
}

Example 18 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class PerformPutHintedHandoff method execute.

@Override
public void execute(Pipeline pipeline) {
    Versioned<byte[]> versionedCopy = pipelineData.getVersionedCopy();
    for (Node slopFinalDestinationNode : pipelineData.getSynchronizer().getDelegatedSlopDestinations()) {
        int failedNodeId = slopFinalDestinationNode.getId();
        if (versionedCopy == null) {
            VectorClock clock = (VectorClock) versioned.getVersion();
            versionedCopy = new Versioned<byte[]>(versioned.getValue(), clock.incremented(failedNodeId, time.getMilliseconds()));
        }
        Version version = versionedCopy.getVersion();
        if (logger.isTraceEnabled())
            logger.trace("Performing parallel hinted handoff for node " + slopFinalDestinationNode + ", store " + pipelineData.getStoreName() + " key " + key + ", version " + version);
        Slop slop = new Slop(pipelineData.getStoreName(), Slop.Operation.PUT, key, versionedCopy.getValue(), transforms, failedNodeId, new Date());
        hintedHandoff.sendHintParallel(slopFinalDestinationNode, version, slop);
    }
    pipeline.addEvent(completeEvent);
}

Example 19 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class SlopPurgeTest method populateSlops.

private void populateSlops(List<Versioned<Slop>> slops) {
    for (Versioned<Slop> slop : slops) {
        VectorClock clock = TestUtils.getClock(1);
        NodeValue<ByteArray, byte[]> nodeValue = new NodeValue<ByteArray, byte[]>(PURGE_SERVER_ID, slop.getValue().makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop.getValue()), clock));
        adminClient.storeOps.putNodeKeyValue("slop", nodeValue);
    }
}

Example 20 with Slop

use of voldemort.store.slop.Slop in project voldemort by voldemort.

the class StreamingSlopPusherTest method testFailedServer.

@Test
public void testFailedServer() throws IOException, InterruptedException {
    startServers(0, 2);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet1 = ServerTestUtils.createRandomSlops(1, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    // Generate slops for 2
    final List<Versioned<Slop>> entrySet2 = ServerTestUtils.createRandomSlops(2, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet1, entrySet2);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(2000);
    // Now check if the slops went through and also got deleted
    Iterator<Versioned<Slop>> entryIterator = entrySet2.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(2).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    entryIterator = entrySet1.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        // did it get deleted correctly
        assertNotSame("slop should be there", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 50);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(50));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    stopServers(0, 2);
}