Examples with VectorClock voldemort.versioning.VectorClock used on opensource projects

Example 56 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class PerformSerialPutRequests method execute.

public void execute(Pipeline pipeline) {
    int currentNode = 0;
    List<Node> nodes = pipelineData.getNodes();
    long startMasterMs = -1;
    long startMasterNs = -1;
    if (logger.isDebugEnabled()) {
        startMasterMs = System.currentTimeMillis();
        startMasterNs = System.nanoTime();
    }
    if (logger.isDebugEnabled())
        logger.debug("Performing serial put requests to determine master");
    Node node = null;
    for (; currentNode < nodes.size(); currentNode++) {
        node = nodes.get(currentNode);
        pipelineData.incrementNodeIndex();
        VectorClock versionedClock = (VectorClock) versioned.getVersion();
        final Versioned<byte[]> versionedCopy = new Versioned<byte[]>(versioned.getValue(), versionedClock.incremented(node.getId(), time.getMilliseconds()));
        if (logger.isDebugEnabled())
            logger.debug("Attempt #" + (currentNode + 1) + " to perform put (node " + node.getId() + ")");
        long start = System.nanoTime();
        try {
            stores.get(node.getId()).put(key, versionedCopy, transforms);
            long requestTime = (System.nanoTime() - start) / Time.NS_PER_MS;
            pipelineData.incrementSuccesses();
            failureDetector.recordSuccess(node, requestTime);
            if (logger.isDebugEnabled())
                logger.debug("Put on node " + node.getId() + " succeeded, using as master");
            pipelineData.setMaster(node);
            pipelineData.setVersionedCopy(versionedCopy);
            pipelineData.getZoneResponses().add(node.getZoneId());
            currentNode++;
            break;
        } catch (Exception e) {
            long requestTime = (System.nanoTime() - start) / Time.NS_PER_MS;
            if (logger.isDebugEnabled())
                logger.debug("Master PUT at node " + currentNode + "(" + node.getHost() + ")" + " failed (" + e.getMessage() + ") in " + (System.nanoTime() - start) + " ns" + " (keyRef: " + System.identityHashCode(key) + ")");
            if (PipelineRoutedStore.isSlopableFailure(e)) {
                pipelineData.getSynchronizer().tryDelegateSlop(node);
            }
            if (handleResponseError(e, node, requestTime, pipeline, failureDetector))
                return;
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("PUT {key:" + key + "} currentNode=" + currentNode + " nodes.size()=" + nodes.size());
    }
    if (pipelineData.getSuccesses() < 1) {
        List<Exception> failures = pipelineData.getFailures();
        pipelineData.setFatalError(new InsufficientOperationalNodesException("No master node succeeded!", failures.size() > 0 ? failures.get(0) : null));
        pipeline.abort();
        return;
    }
    // There aren't any more requests to make...
    if (currentNode == nodes.size()) {
        if (pipelineData.getSuccesses() < required) {
            pipelineData.setFatalError(new InsufficientOperationalNodesException(required + " " + pipeline.getOperation().getSimpleName() + "s required, but only " + pipelineData.getSuccesses() + " succeeded", pipelineData.getReplicationSet(), pipelineData.getNodes(), pipelineData.getFailedNodes(), pipelineData.getFailures()));
            pipeline.abort();
        } else {
            if (pipelineData.getZonesRequired() != null) {
                int zonesSatisfied = pipelineData.getZoneResponses().size();
                if (zonesSatisfied >= (pipelineData.getZonesRequired() + 1)) {
                    pipeline.addEvent(completeEvent);
                } else {
                    pipelineData.setFatalError(new InsufficientZoneResponsesException((pipelineData.getZonesRequired() + 1) + " " + pipeline.getOperation().getSimpleName() + "s required zone, but only " + zonesSatisfied + " succeeded"));
                    pipeline.abort();
                }
            } else {
                if (logger.isDebugEnabled())
                    logger.debug("Finished master PUT for key " + ByteUtils.toHexString(key.get()) + " (keyRef: " + System.identityHashCode(key) + "); started at " + startMasterMs + " took " + (System.nanoTime() - startMasterNs) + " ns on node " + (node == null ? "NULL" : node.getId()) + "(" + (node == null ? "NULL" : node.getHost()) + "); now complete");
                pipeline.addEvent(completeEvent);
            }
        }
    } else {
        if (logger.isDebugEnabled())
            logger.debug("Finished master PUT for key " + ByteUtils.toHexString(key.get()) + " (keyRef: " + System.identityHashCode(key) + "); started at " + startMasterMs + " took " + (System.nanoTime() - startMasterNs) + " ns on node " + (node == null ? "NULL" : node.getId()) + "(" + (node == null ? "NULL" : node.getHost()) + ")");
        pipeline.addEvent(masterDeterminedEvent);
    }
}

Example 57 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class GossiperTest method testGossiper.

// Protect against this test running forever until the root cause of running
// forever is found.
@Test(timeout = 1800)
public void testGossiper() throws Exception {
    Cluster newCluster = null;
    boolean startedAdditionalServer = false;
    while (!startedAdditionalServer) {
        try {
            newCluster = attemptStartAdditionalServer();
            startedAdditionalServer = true;
        } catch (IOException ioe) {
            logger.warn("Caught an IOException when attempting to start additional server. Will print stacktrace and then attempt to start additional server again.");
            ioe.printStackTrace();
        }
    }
    // Get the new cluster.xml
    AdminClient localAdminClient = getAdminClient(newCluster);
    Versioned<String> versionedClusterXML = localAdminClient.metadataMgmtOps.getRemoteMetadata(3, MetadataStore.CLUSTER_KEY);
    // Increment the version, let what would be the "donor node" know about
    // it to seed the Gossip.
    Version version = versionedClusterXML.getVersion();
    ((VectorClock) version).incrementVersion(3, ((VectorClock) version).getTimestamp() + 1);
    ((VectorClock) version).incrementVersion(0, ((VectorClock) version).getTimestamp() + 1);
    localAdminClient.metadataMgmtOps.updateRemoteMetadata(0, MetadataStore.CLUSTER_KEY, versionedClusterXML);
    localAdminClient.metadataMgmtOps.updateRemoteMetadata(3, MetadataStore.CLUSTER_KEY, versionedClusterXML);
    try {
        Thread.sleep(500);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    // Wait up to five seconds for Gossip to spread
    final Cluster newFinalCluster = newCluster;
    try {
        TestUtils.assertWithBackoff(5000, new Attempt() {

            public void checkCondition() {
                int serversSeen = 0;
                // Now verify that we have gossiped correctly
                for (VoldemortServer server : servers) {
                    Cluster clusterAtServer = server.getMetadataStore().getCluster();
                    int nodeId = server.getMetadataStore().getNodeId();
                    assertEquals("server " + nodeId + " has heard " + " the gossip about number of nodes", clusterAtServer.getNumberOfNodes(), newFinalCluster.getNumberOfNodes());
                    assertEquals("server " + nodeId + " has heard " + " the gossip about partitions", clusterAtServer.getNodeById(nodeId).getPartitionIds(), newFinalCluster.getNodeById(nodeId).getPartitionIds());
                    serversSeen++;
                }
                assertEquals("saw all servers", serversSeen, servers.size());
            }
        });
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
}

Example 58 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class ExceededQuotaSlopTest method setGetPutQuotasForEachServer.

public void setGetPutQuotasForEachServer() throws Exception {
    Properties adminProperties = new Properties();
    adminProperties.setProperty("max_connections", "2");
    adminClient = new AdminClient(cluster, new AdminClientConfig().setMaxConnectionsPerNode(2));
    Map<Pair<Integer, QuotaType>, Integer> throughPutMap = new HashMap<Pair<Integer, QuotaType>, Integer>();
    // Set Node0 Quota
    throughPutMap.put(new Pair<Integer, QuotaType>(0, QuotaType.PUT_THROUGHPUT), 5);
    throughPutMap.put(new Pair<Integer, QuotaType>(0, QuotaType.GET_THROUGHPUT), 20);
    // Set Node1 Quota
    throughPutMap.put(new Pair<Integer, QuotaType>(1, QuotaType.PUT_THROUGHPUT), 2);
    throughPutMap.put(new Pair<Integer, QuotaType>(1, QuotaType.GET_THROUGHPUT), 20);
    for (Entry<Pair<Integer, QuotaType>, Integer> throughPut : throughPutMap.entrySet()) {
        int nodeId = throughPut.getKey().getFirst();
        QuotaType type = throughPut.getKey().getSecond();
        int value = throughPut.getValue();
        VectorClock clock = VectorClockUtils.makeClockWithCurrentTime(cluster.getNodeIds());
        NodeValue<ByteArray, byte[]> operationValue = new NodeValue<ByteArray, byte[]>(nodeId, new ByteArray(getKeyBytes(type)), new Versioned<byte[]>(ByteUtils.getBytes(Integer.toString(value), encodingType), clock));
        try {
            adminClient.storeOps.putNodeKeyValue(quotaStoreName, operationValue);
        } catch (Exception e) {
            throw new Exception("Exception when setting put quota for node " + nodeId + " Operation " + type + "." + e.getMessage());
        }
    }
}

Example 59 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class AbstractRequestFormatTest method testGetVersionRequest.

public void testGetVersionRequest(ByteArray key, byte[] value, VectorClock version, boolean isPresent) throws Exception {
    try {
        if (isPresent) {
            testPutRequest(key, value, null, version, null, false);
        }
        ByteArrayOutputStream getVersionRequest = new ByteArrayOutputStream();
        this.clientWireFormat.writeGetVersionRequest(new DataOutputStream(getVersionRequest), storeName, key, RequestRoutingType.NORMAL);
        ByteArrayOutputStream getVersionResponse = handleRequest(getVersionRequest);
        testIsCompleteGetVersionResponse(getVersionResponse);
        List<Version> values = this.clientWireFormat.readGetVersionResponse(inputStream(getVersionResponse));
        if (isPresent) {
            assertEquals(1, values.size());
            VectorClock returnValue = (VectorClock) values.get(0);
            assertEquals(version, returnValue);
            assertEquals(version.getTimestamp(), returnValue.getTimestamp());
        } else {
            assertEquals(0, values.size());
        }
    } finally {
        this.store.deleteAll();
    }
}

Example 60 with VectorClock

use of voldemort.versioning.VectorClock in project voldemort by voldemort.

the class DataCleanupJobTest method updateStoreDef.

private void updateStoreDef(int retentionDays) {
    StoreDefinition storeDef = getStoreDef(retentionDays);
    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
    String storeStr = mapper.writeStoreList(Arrays.asList(storeDef));
    VectorClock clock = new VectorClock(System.currentTimeMillis());
    clock.incrementVersion(0, System.currentTimeMillis());
    Versioned<byte[]> storeSerialized = new Versioned<byte[]>(ByteUtils.getBytes(storeStr, "UTF-8"), clock);
    metadataStore.updateStoreDefinitions(storeSerialized);
}