Search in sources :

Example 91 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class StreamingSlopPusherTest method testPushForHostSwap.

/**
     * 
     * Procedure: 1. bring up 2-node cluster; 2. set up slop pusher job with the
     * cluster info; 3. replace node 1 by simply changing its port-ids, update
     * the cluster metadata to whole cluster; 4. run slop pusher job and it has
     * to fail; 5. setCluster in FD and the slop pusher job has to succeed
     * 
     * @throws InterruptedException
     * @throws IOException
     */
@Test
public void testPushForHostSwap() throws InterruptedException, IOException {
    startServers(0, 1);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet = ServerTestUtils.createRandomSlops(1, 100, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet);
    // replace node 1 and update metadata on all nodes
    replaceOneNode(1);
    Cluster oldCluster = cluster;
    // test if node 1 is down
    stopServersWithoutRemovingVoldemortHome(1);
    // slop should fail here
    new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new ThresholdFailureDetector(new FailureDetectorConfig().setCluster(oldCluster).setConnectionVerifier(new AdminSlopStreamingVerifier(oldCluster))), configs[0], new ScanPermitWrapper(1)).run();
    Thread.sleep(2000);
    assertFalse(verifySlopPushResult(entrySet, slopStoreNode0, 1));
    // test if node 1 is up
    startServers(1);
    // slop push should fail
    new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new ThresholdFailureDetector(new FailureDetectorConfig().setCluster(oldCluster).setConnectionVerifier(new AdminSlopStreamingVerifier(oldCluster))), configs[0], new ScanPermitWrapper(1)).run();
    Thread.sleep(2000);
    Assert.assertTrue(verifySlopPushResult(entrySet, slopStoreNode0, 1));
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 0);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(0));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    stopServers(0, 1);
}
Also used : ThresholdFailureDetector(voldemort.cluster.failuredetector.ThresholdFailureDetector) Versioned(voldemort.versioning.Versioned) ScanPermitWrapper(voldemort.server.storage.ScanPermitWrapper) Cluster(voldemort.cluster.Cluster) StreamingSlopPusherJob(voldemort.server.scheduler.slop.StreamingSlopPusherJob) AdminSlopStreamingVerifier(voldemort.cluster.failuredetector.AdminSlopStreamingVerifier) FailureDetectorConfig(voldemort.cluster.failuredetector.FailureDetectorConfig) ByteArray(voldemort.utils.ByteArray) SlopStorageEngine(voldemort.store.slop.SlopStorageEngine) Slop(voldemort.store.slop.Slop) Test(org.junit.Test)

Example 92 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class StreamingSlopPusherTest method testServerReplacementWithoutBounce.

@Test
public void testServerReplacementWithoutBounce() throws IOException, InterruptedException {
    startServers(0, 2);
    // Put into slop store 0
    StorageEngine<ByteArray, Slop, byte[]> slopStoreNode0 = getVoldemortServer(0).getStoreRepository().getSlopStore().asSlopStore();
    // Generate slops for 1
    final List<Versioned<Slop>> entrySet1 = ServerTestUtils.createRandomSlops(1, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    // Generate slops for 2
    final List<Versioned<Slop>> entrySet2 = ServerTestUtils.createRandomSlops(2, 50, "test-replication-memory", "users", "test-replication-persistent", "test-readrepair-memory", "test-consistent", "test-consistent-with-pref-list");
    populateSlops(0, slopStoreNode0, entrySet1, entrySet2);
    StreamingSlopPusherJob pusher = new StreamingSlopPusherJob(getVoldemortServer(0).getStoreRepository(), getVoldemortServer(0).getMetadataStore(), new BannagePeriodFailureDetector(new FailureDetectorConfig().setCluster(cluster).setConnectionVerifier(new ServerStoreConnectionVerifier(socketStoreFactory, metadataStore, configs[0]))), configs[0], new ScanPermitWrapper(1));
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(10000);
    // Now check if the slops went through and also got deleted
    Iterator<Versioned<Slop>> entryIterator = entrySet2.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(2).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    entryIterator = entrySet1.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        // did it get deleted correctly
        assertNotSame("slop should be there", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
    // Check counts
    SlopStorageEngine slopEngine = getVoldemortServer(0).getStoreRepository().getSlopStore();
    assertEquals(slopEngine.getOutstandingTotal(), 50);
    assertEquals(slopEngine.getOutstandingByNode().get(1), new Long(50));
    assertEquals(slopEngine.getOutstandingByNode().get(2), new Long(0));
    // now replace server 1 with a new host and start it
    cluster = ServerTestUtils.updateClusterWithNewHost(cluster, 1);
    startServers(1);
    // update the meatadata store with the new cluster on node 0 and 2 (the
    // two servers that are running)
    servers[0].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);
    servers[2].getMetadataStore().put(MetadataStore.CLUSTER_KEY, cluster);
    // Give some time for the pusher job to figure out that server1 is up
    Thread.sleep(35000);
    // start the pusher job again
    pusher.run();
    // Give some time for the slops to go over
    Thread.sleep(10000);
    // make sure the slot for server 1 is pushed to the new host
    // Now check if the slops went through and also got deleted
    entryIterator = entrySet1.listIterator();
    while (entryIterator.hasNext()) {
        Versioned<Slop> versionedSlop = entryIterator.next();
        Slop nextSlop = versionedSlop.getValue();
        StorageEngine<ByteArray, byte[], byte[]> store = getVoldemortServer(1).getStoreRepository().getStorageEngine(nextSlop.getStoreName());
        if (nextSlop.getOperation().equals(Slop.Operation.PUT)) {
            assertNotSame("entry should be present at store", 0, store.get(nextSlop.getKey(), null).size());
            assertEquals("entry value should match", new String(nextSlop.getValue()), new String(store.get(nextSlop.getKey(), null).get(0).getValue()));
        } else if (nextSlop.getOperation().equals(Slop.Operation.DELETE)) {
            assertEquals("entry value should match", 0, store.get(nextSlop.getKey(), null).size());
        }
        // did it get deleted correctly
        assertEquals("slop should have gone", 0, slopStoreNode0.get(nextSlop.makeKey(), null).size());
    }
}
Also used : Versioned(voldemort.versioning.Versioned) ScanPermitWrapper(voldemort.server.storage.ScanPermitWrapper) StreamingSlopPusherJob(voldemort.server.scheduler.slop.StreamingSlopPusherJob) FailureDetectorConfig(voldemort.cluster.failuredetector.FailureDetectorConfig) ByteArray(voldemort.utils.ByteArray) ServerStoreConnectionVerifier(voldemort.cluster.failuredetector.ServerStoreConnectionVerifier) SlopStorageEngine(voldemort.store.slop.SlopStorageEngine) Slop(voldemort.store.slop.Slop) BannagePeriodFailureDetector(voldemort.cluster.failuredetector.BannagePeriodFailureDetector) Test(org.junit.Test)

Example 93 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class DataCleanupJobTest method updateStoreDef.

private void updateStoreDef(int retentionDays) {
    StoreDefinition storeDef = getStoreDef(retentionDays);
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    String storeStr = mapper.writeStoreList(Arrays.asList(storeDef));
    VectorClock clock = new VectorClock(System.currentTimeMillis());
    clock.incrementVersion(0, System.currentTimeMillis());
    Versioned<byte[]> storeSerialized = new Versioned<byte[]>(ByteUtils.getBytes(storeStr, "UTF-8"), clock);
    metadataStore.updateStoreDefinitions(storeSerialized);
}
Also used : Versioned(voldemort.versioning.Versioned) StoreDefinition(voldemort.store.StoreDefinition) VectorClock(voldemort.versioning.VectorClock) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper)

Example 94 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class DataCleanupJobTest method assertAbsence.

private void assertAbsence(int start, int end) {
    for (int i = start; i < end; i++) {
        ByteArray b = new ByteArray(Integer.toString(i).getBytes());
        List<Versioned<byte[]>> found = engine.get(b, null);
        assertTrue("Expected key '" + i + "' to be deleted!", found.size() == 0);
    }
}
Also used : Versioned(voldemort.versioning.Versioned) ByteArray(voldemort.utils.ByteArray)

Example 95 with Versioned

use of voldemort.versioning.Versioned in project voldemort by voldemort.

the class SlopPusherDeadSlopTest method testAutoPurge.

@Test
public void testAutoPurge() {
    try {
        // generate slops for a non existent node 2.
        List<Versioned<Slop>> deadNodeSlops = ServerTestUtils.createRandomSlops(2, 40, false, "test");
        // generate slops for a non existent store "deleted_store"
        List<Versioned<Slop>> deadStoreSlops = ServerTestUtils.createRandomSlops(0, 40, false, "deleted_store");
        // generate some valid slops and make sure they go into the
        // destination store
        List<Versioned<Slop>> validStoreSlops = ServerTestUtils.createRandomSlops(1, 40, false, "test");
        List<Versioned<Slop>> slops = new ArrayList<Versioned<Slop>>();
        slops.addAll(deadStoreSlops);
        slops.addAll(deadNodeSlops);
        slops.addAll(validStoreSlops);
        SlopSerializer slopSerializer = new SlopSerializer();
        // Populate the store with the slops
        for (Versioned<Slop> slop : slops) {
            VectorClock clock = TestUtils.getClock(1);
            NodeValue<ByteArray, byte[]> nodeValue = new NodeValue<ByteArray, byte[]>(0, slop.getValue().makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop.getValue()), clock));
            adminClient.storeOps.putNodeKeyValue("slop", nodeValue);
        }
        // wait for twice the slop interval (in case a slop push was
        // underway as we populated)
        Thread.sleep(SLOP_FREQUENCY_MS * 2);
        // Confirm the dead slops are all gone now..
        for (List<Versioned<Slop>> deadSlops : Arrays.asList(deadStoreSlops, deadNodeSlops)) {
            for (Versioned<Slop> slop : deadSlops) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("slop", 0, slop.getValue().makeKey());
                assertEquals("Slop should be purged", 0, slopEntry.size());
            }
        }
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getSingleStoreDefinitionsXml()));
        BaseStoreRoutingPlan rPlan = new BaseStoreRoutingPlan(adminClient.getAdminClientCluster(), StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, "test"));
        // Confirm the valid ones made it
        for (Versioned<Slop> slop : validStoreSlops) {
            ByteArray key = slop.getValue().getKey();
            if (rPlan.getReplicationNodeList(key.get()).contains(1)) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("test", 1, key);
                if (slop.getValue().getOperation() == Operation.DELETE) {
                    assertTrue("Delete Slop should have not reached destination", slopEntry.size() == 0);
                } else {
                    assertTrue("Put Slop should have reached destination", slopEntry.size() > 0);
                }
            }
        }
    } catch (Exception e) {
        logger.error("Test failed with", e);
        fail("unexpected exception");
    }
}
Also used : NodeValue(voldemort.store.routed.NodeValue) Versioned(voldemort.versioning.Versioned) VectorClock(voldemort.versioning.VectorClock) ArrayList(java.util.ArrayList) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) IOException(java.io.IOException) StoreDefinition(voldemort.store.StoreDefinition) StringReader(java.io.StringReader) ByteArray(voldemort.utils.ByteArray) BaseStoreRoutingPlan(voldemort.routing.BaseStoreRoutingPlan) Slop(voldemort.store.slop.Slop) SlopSerializer(voldemort.serialization.SlopSerializer) Test(org.junit.Test)

Aggregations

Versioned (voldemort.versioning.Versioned)214 ByteArray (voldemort.utils.ByteArray)130 Test (org.junit.Test)88 VectorClock (voldemort.versioning.VectorClock)73 ArrayList (java.util.ArrayList)56 VoldemortException (voldemort.VoldemortException)48 List (java.util.List)36 StoreDefinition (voldemort.store.StoreDefinition)29 HashMap (java.util.HashMap)28 IOException (java.io.IOException)27 Node (voldemort.cluster.Node)27 Pair (voldemort.utils.Pair)25 Slop (voldemort.store.slop.Slop)22 ObsoleteVersionException (voldemort.versioning.ObsoleteVersionException)22 Map (java.util.Map)21 Cluster (voldemort.cluster.Cluster)17 AbstractByteArrayStoreTest (voldemort.store.AbstractByteArrayStoreTest)15 InvalidMetadataException (voldemort.store.InvalidMetadataException)13 PersistenceFailureException (voldemort.store.PersistenceFailureException)13 Version (voldemort.versioning.Version)13