Examples with VectorClock voldemort.versioning.VectorClock used on opensource projects

Example 61 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class GossiperTest method testGossiper.

// Protect against this test running forever until the root cause of running
// forever is found.
@Test(timeout = 1800)
public void testGossiper() throws Exception {
    Cluster newCluster = null;
    boolean startedAdditionalServer = false;
    while (!startedAdditionalServer) {
        try {
            newCluster = attemptStartAdditionalServer();
            startedAdditionalServer = true;
        } catch (IOException ioe) {
            logger.warn("Caught an IOException when attempting to start additional server. Will print stacktrace and then attempt to start additional server again.");
            ioe.printStackTrace();
        }
    }
    // Get the new cluster.xml
    AdminClient localAdminClient = getAdminClient(newCluster);
    Versioned<String> versionedClusterXML = localAdminClient.metadataMgmtOps.getRemoteMetadata(3, MetadataStore.CLUSTER_KEY);
    // Increment the version, let what would be the "donor node" know about
    // it to seed the Gossip.
    Version version = versionedClusterXML.getVersion();
    ((VectorClock) version).incrementVersion(3, ((VectorClock) version).getTimestamp() + 1);
    ((VectorClock) version).incrementVersion(0, ((VectorClock) version).getTimestamp() + 1);
    localAdminClient.metadataMgmtOps.updateRemoteMetadata(0, MetadataStore.CLUSTER_KEY, versionedClusterXML);
    localAdminClient.metadataMgmtOps.updateRemoteMetadata(3, MetadataStore.CLUSTER_KEY, versionedClusterXML);
    try {
        Thread.sleep(500);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    // Wait up to five seconds for Gossip to spread
    final Cluster newFinalCluster = newCluster;
    try {
        TestUtils.assertWithBackoff(5000, new Attempt() {

            public void checkCondition() {
                int serversSeen = 0;
                // Now verify that we have gossiped correctly
                for (VoldemortServer server : servers) {
                    Cluster clusterAtServer = server.getMetadataStore().getCluster();
                    int nodeId = server.getMetadataStore().getNodeId();
                    assertEquals("server " + nodeId + " has heard " + " the gossip about number of nodes", clusterAtServer.getNumberOfNodes(), newFinalCluster.getNumberOfNodes());
                    assertEquals("server " + nodeId + " has heard " + " the gossip about partitions", clusterAtServer.getNodeById(nodeId).getPartitionIds(), newFinalCluster.getNodeById(nodeId).getPartitionIds());
                    serversSeen++;
                }
                assertEquals("saw all servers", serversSeen, servers.size());
            }
        });
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
}

Example 62 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class ExceededQuotaSlopTest method setGetPutQuotasForEachServer.

public void setGetPutQuotasForEachServer() throws Exception {
    Properties adminProperties = new Properties();
    adminProperties.setProperty("max_connections", "2");
    adminClient = new AdminClient(cluster, new AdminClientConfig().setMaxConnectionsPerNode(2));
    Map<Pair<Integer, QuotaType>, Integer> throughPutMap = new HashMap<Pair<Integer, QuotaType>, Integer>();
    // Set Node0 Quota
    throughPutMap.put(new Pair<Integer, QuotaType>(0, QuotaType.PUT_THROUGHPUT), 5);
    throughPutMap.put(new Pair<Integer, QuotaType>(0, QuotaType.GET_THROUGHPUT), 20);
    // Set Node1 Quota
    throughPutMap.put(new Pair<Integer, QuotaType>(1, QuotaType.PUT_THROUGHPUT), 2);
    throughPutMap.put(new Pair<Integer, QuotaType>(1, QuotaType.GET_THROUGHPUT), 20);
    for (Entry<Pair<Integer, QuotaType>, Integer> throughPut : throughPutMap.entrySet()) {
        int nodeId = throughPut.getKey().getFirst();
        QuotaType type = throughPut.getKey().getSecond();
        int value = throughPut.getValue();
        VectorClock clock = VectorClockUtils.makeClockWithCurrentTime(cluster.getNodeIds());
        NodeValue<ByteArray, byte[]> operationValue = new NodeValue<ByteArray, byte[]>(nodeId, new ByteArray(getKeyBytes(type)), new Versioned<byte[]>(ByteUtils.getBytes(Integer.toString(value), encodingType), clock));
        try {
            adminClient.storeOps.putNodeKeyValue(quotaStoreName, operationValue);
        } catch (Exception e) {
            throw new Exception("Exception when setting put quota for node " + nodeId + " Operation " + type + "." + e.getMessage());
        }
    }
}

Example 63 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class AbstractRequestFormatTest method testGetVersionRequest.

public void testGetVersionRequest(ByteArray key, byte[] value, VectorClock version, boolean isPresent) throws Exception {
    try {
        if (isPresent) {
            testPutRequest(key, value, null, version, null, false);
        }
        ByteArrayOutputStream getVersionRequest = new ByteArrayOutputStream();
        this.clientWireFormat.writeGetVersionRequest(new DataOutputStream(getVersionRequest), storeName, key, RequestRoutingType.NORMAL);
        ByteArrayOutputStream getVersionResponse = handleRequest(getVersionRequest);
        testIsCompleteGetVersionResponse(getVersionResponse);
        List<Version> values = this.clientWireFormat.readGetVersionResponse(inputStream(getVersionResponse));
        if (isPresent) {
            assertEquals(1, values.size());
            VectorClock returnValue = (VectorClock) values.get(0);
            assertEquals(version, returnValue);
            assertEquals(version.getTimestamp(), returnValue.getTimestamp());
        } else {
            assertEquals(0, values.size());
        }
    } finally {
        this.store.deleteAll();
    }
}

Example 64 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class DataCleanupJobTest method updateStoreDef.

private void updateStoreDef(int retentionDays) {
    StoreDefinition storeDef = getStoreDef(retentionDays);
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    String storeStr = mapper.writeStoreList(Arrays.asList(storeDef));
    VectorClock clock = new VectorClock(System.currentTimeMillis());
    clock.incrementVersion(0, System.currentTimeMillis());
    Versioned<byte[]> storeSerialized = new Versioned<byte[]>(ByteUtils.getBytes(storeStr, "UTF-8"), clock);
    metadataStore.updateStoreDefinitions(storeSerialized);
}

Example 65 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class SlopPusherDeadSlopTest method testAutoPurge.

@Test
public void testAutoPurge() {
    try {
        // generate slops for a non existent node 2.
        List<Versioned<Slop>> deadNodeSlops = ServerTestUtils.createRandomSlops(2, 40, false, "test");
        // generate slops for a non existent store "deleted_store"
        List<Versioned<Slop>> deadStoreSlops = ServerTestUtils.createRandomSlops(0, 40, false, "deleted_store");
        // generate some valid slops and make sure they go into the
        // destination store
        List<Versioned<Slop>> validStoreSlops = ServerTestUtils.createRandomSlops(1, 40, false, "test");
        List<Versioned<Slop>> slops = new ArrayList<Versioned<Slop>>();
        slops.addAll(deadStoreSlops);
        slops.addAll(deadNodeSlops);
        slops.addAll(validStoreSlops);
        SlopSerializer slopSerializer = new SlopSerializer();
        // Populate the store with the slops
        for (Versioned<Slop> slop : slops) {
            VectorClock clock = TestUtils.getClock(1);
            NodeValue<ByteArray, byte[]> nodeValue = new NodeValue<ByteArray, byte[]>(0, slop.getValue().makeKey(), new Versioned<byte[]>(slopSerializer.toBytes(slop.getValue()), clock));
            adminClient.storeOps.putNodeKeyValue("slop", nodeValue);
        }
        // wait for twice the slop interval (in case a slop push was
        // underway as we populated)
        Thread.sleep(SLOP_FREQUENCY_MS * 2);
        // Confirm the dead slops are all gone now..
        for (List<Versioned<Slop>> deadSlops : Arrays.asList(deadStoreSlops, deadNodeSlops)) {
            for (Versioned<Slop> slop : deadSlops) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("slop", 0, slop.getValue().makeKey());
                assertEquals("Slop should be purged", 0, slopEntry.size());
            }
        }
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        List<StoreDefinition> storeDefs = mapper.readStoreList(new StringReader(VoldemortTestConstants.getSingleStoreDefinitionsXml()));
        BaseStoreRoutingPlan rPlan = new BaseStoreRoutingPlan(adminClient.getAdminClientCluster(), StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, "test"));
        // Confirm the valid ones made it
        for (Versioned<Slop> slop : validStoreSlops) {
            ByteArray key = slop.getValue().getKey();
            if (rPlan.getReplicationNodeList(key.get()).contains(1)) {
                List<Versioned<byte[]>> slopEntry = adminClient.storeOps.getNodeKey("test", 1, key);
                if (slop.getValue().getOperation() == Operation.DELETE) {
                    assertTrue("Delete Slop should have not reached destination", slopEntry.size() == 0);
                } else {
                    assertTrue("Put Slop should have reached destination", slopEntry.size() > 0);
                }
            }
        }
    } catch (Exception e) {
        logger.error("Test failed with", e);
        fail("unexpected exception");
    }
}