Search in sources :

Example 1 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AbstractRebalanceTest method checkForKeyExistence.

/**
 * REFACTOR: these should belong AdminClient so existence checks can be done
 * easily across the board
 *
 * @param admin
 * @param serverId
 * @param store
 * @param keyList
 */
protected void checkForKeyExistence(AdminClient admin, int serverId, String store, List<ByteArray> keyList) {
    // do the positive tests
    Iterator<QueryKeyResult> positiveTestResultsItr = admin.streamingOps.queryKeys(serverId, store, keyList.iterator());
    while (positiveTestResultsItr.hasNext()) {
        QueryKeyResult item = positiveTestResultsItr.next();
        ByteArray key = item.getKey();
        List<Versioned<byte[]>> vals = item.getValues();
        Exception e = item.getException();
        assertEquals("Error fetching key " + key, null, e);
        assertEquals("Value not found for key " + key, true, vals != null & vals.size() != 0);
    }
}
Also used : Versioned(voldemort.versioning.Versioned) ByteArray(voldemort.utils.ByteArray) VoldemortException(voldemort.VoldemortException) QueryKeyResult(voldemort.client.protocol.admin.QueryKeyResult)

Example 2 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AbstractRebalanceTest method checkForTupleEquivalence.

/**
 * REFACTOR: these should belong AdminClient so existence checks can be done
 * easily across the board
 *
 * @param admin
 * @param serverId
 * @param store
 * @param keyList
 */
protected void checkForTupleEquivalence(AdminClient admin, int serverId, String store, List<ByteArray> keyList, HashMap<String, String> baselineTuples, HashMap<String, VectorClock> baselineVersions) {
    // do the positive tests
    Iterator<QueryKeyResult> positiveTestResultsItr = admin.streamingOps.queryKeys(serverId, store, keyList.iterator());
    while (positiveTestResultsItr.hasNext()) {
        QueryKeyResult item = positiveTestResultsItr.next();
        ByteArray key = item.getKey();
        List<Versioned<byte[]>> vals = item.getValues();
        Exception e = item.getException();
        assertEquals("Error fetching key " + key, null, e);
        assertEquals("Value not found for key " + key, true, vals != null & vals.size() != 0);
        String keyStr = ByteUtils.getString(key.get(), "UTF-8");
        if (baselineTuples != null)
            assertEquals("Value does not match up ", baselineTuples.get(keyStr), ByteUtils.getString(vals.get(0).getValue(), "UTF-8"));
        if (baselineVersions != null)
            assertEquals("Version does not match up", baselineVersions.get(keyStr), vals.get(0).getVersion());
    }
}
Also used : Versioned(voldemort.versioning.Versioned) ByteArray(voldemort.utils.ByteArray) VoldemortException(voldemort.VoldemortException) QueryKeyResult(voldemort.client.protocol.admin.QueryKeyResult)

Example 3 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AbstractConsistencyFixer method doReads.

/**
 * @param nodeIdList
 * @param keyInBytes
 * @param keyInHexFormat
 * @return
 */
private Map<Integer, QueryKeyResult> doReads(final List<Integer> nodeIdList, final byte[] keyInBytes, final String keyInHexFormat) {
    Map<Integer, QueryKeyResult> nodeIdToKeyValues = new HashMap<Integer, QueryKeyResult>();
    ByteArray key = new ByteArray(keyInBytes);
    for (int nodeId : nodeIdList) {
        List<Versioned<byte[]>> values = null;
        try {
            values = this.adminClient.storeOps.getNodeKey(this.storeInstance.getStoreDefinition().getName(), nodeId, key);
            nodeIdToKeyValues.put(nodeId, new QueryKeyResult(key, values));
        } catch (VoldemortException ve) {
            nodeIdToKeyValues.put(nodeId, new QueryKeyResult(key, ve));
        }
    }
    return nodeIdToKeyValues;
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) VoldemortException(voldemort.VoldemortException) QueryKeyResult(voldemort.client.protocol.admin.QueryKeyResult)

Example 4 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AdminServiceBasicTest method testQuery.

@Test
public void testQuery() {
    HashMap<ByteArray, byte[]> belongToAndInsideServer0 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> belongToAndInsideServer1 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> notBelongServer0ButInsideServer0 = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> belongToServer0ButOutsideBoth = new HashMap<ByteArray, byte[]>();
    HashMap<ByteArray, byte[]> notBelongToServer0AndOutsideBoth = new HashMap<ByteArray, byte[]>();
    Store<ByteArray, byte[], byte[]> store0 = getStore(0, testStoreName);
    Store<ByteArray, byte[], byte[]> store1 = getStore(1, testStoreName);
    HashMap<ByteArray, byte[]> entrySet = null;
    Iterator<ByteArray> keys = null;
    RoutingStrategy strategy = servers[0].getMetadataStore().getRoutingStrategy(testStoreName);
    while (true) {
        ByteArray key;
        byte[] value;
        if (keys == null || !keys.hasNext()) {
            entrySet = ServerTestUtils.createRandomKeyValuePairs(100);
            keys = entrySet.keySet().iterator();
        }
        key = keys.next();
        value = entrySet.get(key);
        List<Node> routedNodes = strategy.routeRequest(key.get());
        boolean keyShouldBeInNode0 = false;
        boolean keyShouldBeInNode1 = false;
        for (Node node : routedNodes) {
            keyShouldBeInNode0 = keyShouldBeInNode0 || (node.getId() == 0);
            keyShouldBeInNode1 = keyShouldBeInNode1 || (node.getId() == 1);
        }
        if (belongToAndInsideServer0.size() < 10) {
            if (keyShouldBeInNode0) {
                belongToAndInsideServer0.put(key, value);
                store0.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (belongToAndInsideServer1.size() < 10) {
            if (keyShouldBeInNode1) {
                belongToAndInsideServer1.put(key, value);
                store1.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (notBelongServer0ButInsideServer0.size() < 5) {
            if (!keyShouldBeInNode0) {
                notBelongServer0ButInsideServer0.put(key, value);
                store0.put(key, new Versioned<byte[]>(value), null);
            }
        } else if (belongToServer0ButOutsideBoth.size() < 5) {
            if (keyShouldBeInNode0) {
                belongToServer0ButOutsideBoth.put(key, value);
            }
        } else if (notBelongToServer0AndOutsideBoth.size() < 5) {
            if (!keyShouldBeInNode0) {
                notBelongToServer0AndOutsideBoth.put(key, value);
            }
        } else {
            break;
        }
    }
    ArrayList<ByteArray> belongToAndInsideServer0Keys = new ArrayList<ByteArray>(belongToAndInsideServer0.keySet());
    ArrayList<ByteArray> belongToAndInsideServer1Keys = new ArrayList<ByteArray>(belongToAndInsideServer1.keySet());
    ArrayList<ByteArray> notBelongServer0ButInsideServer0Keys = new ArrayList<ByteArray>(notBelongServer0ButInsideServer0.keySet());
    ArrayList<ByteArray> belongToServer0ButOutsideBothKeys = new ArrayList<ByteArray>(belongToServer0ButOutsideBoth.keySet());
    ArrayList<ByteArray> notBelongToServer0AndOutsideBothKeys = new ArrayList<ByteArray>(notBelongToServer0AndOutsideBoth.keySet());
    List<ByteArray> queryKeys;
    Iterator<QueryKeyResult> results;
    QueryKeyResult entry;
    // test one key on store 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertEquals(queryKeys.get(0), entry.getKey());
    assertNull("There should not be exception in response", entry.getException());
    assertEquals("There should be only 1 value in versioned list", 1, entry.getValues().size());
    assertEquals("Two byte[] should be equal", 0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(0)), entry.getValues().get(0).getValue()));
    assertFalse("There should be only one result", results.hasNext());
    // test one key belongs to but not exists in server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToServer0ButOutsideBothKeys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertFalse("There should not be exception", entry.hasException());
    assertTrue("There should be values", entry.hasValues());
    assertNotNull("Response should be non-null", entry.getValues());
    assertEquals("Value should be empty list", 0, entry.getValues().size());
    assertNull("There should not be exception", entry.getException());
    // test one key not exist and does not belong to server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(notBelongToServer0AndOutsideBothKeys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // test one key that exists on server 0 but does not belong to server 0
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(0));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // test one key deleted
    store0.delete(belongToAndInsideServer0Keys.get(4), null);
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(4));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertFalse("There should not be exception", entry.hasException());
    assertTrue("There should be values", entry.hasValues());
    assertEquals("Not the right key", queryKeys.get(0), entry.getKey());
    assertEquals("Value should be empty list", 0, entry.getValues().size());
    // test empty request
    queryKeys = new ArrayList<ByteArray>();
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertFalse("Results should be empty", results.hasNext());
    // test null key
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(null);
    assertEquals(1, queryKeys.size());
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    entry = results.next();
    assertFalse("There should not be more results", results.hasNext());
    assertTrue("There should be exception", entry.hasException());
    assertFalse("There should not be values", entry.hasValues());
    assertNull("Value should be null", entry.getValues());
    assertTrue("There should be IllegalArgumentException exception", entry.getException() instanceof IllegalArgumentException);
    // test multiple keys (3) on store 1
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer1Keys.get(0));
    queryKeys.add(belongToAndInsideServer1Keys.get(1));
    queryKeys.add(belongToAndInsideServer1Keys.get(2));
    results = getAdminClient().streamingOps.queryKeys(1, testStoreName, queryKeys.iterator());
    assertTrue("Results should not be empty", results.hasNext());
    Map<ByteArray, List<Versioned<byte[]>>> entries = new HashMap<ByteArray, List<Versioned<byte[]>>>();
    int resultCount = 0;
    while (results.hasNext()) {
        resultCount++;
        entry = results.next();
        assertNull("There should not be exception in response", entry.getException());
        assertNotNull("Value should not be null for Key: ", entry.getValues());
        entries.put(entry.getKey(), entry.getValues());
    }
    assertEquals("There should 3 and only 3 results", 3, resultCount);
    for (ByteArray key : queryKeys) {
        // this loop and the count ensure one-to-one mapping
        assertNotNull("This key should exist in the results: " + key, entries.get(key));
        assertEquals("Two byte[] should be equal for key: " + key, 0, ByteUtils.compare(belongToAndInsideServer1.get(key), entries.get(key).get(0).getValue()));
    }
    // test multiple keys, mixed situation
    // key 0: Exists and belongs to
    // key 1: Exists but does not belong to
    // key 2: Does not exist but belongs to
    // key 3: Does not belong and not exist
    // key 4: Same situation with key0
    // key 5: Deleted
    // key 6: Same situation with key2
    store0.delete(belongToAndInsideServer0Keys.get(5), null);
    queryKeys = new ArrayList<ByteArray>();
    queryKeys.add(belongToAndInsideServer0Keys.get(2));
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(1));
    queryKeys.add(belongToServer0ButOutsideBothKeys.get(1));
    queryKeys.add(notBelongToServer0AndOutsideBothKeys.get(1));
    queryKeys.add(belongToAndInsideServer0Keys.get(3));
    queryKeys.add(belongToAndInsideServer0Keys.get(5));
    queryKeys.add(notBelongServer0ButInsideServer0Keys.get(2));
    results = getAdminClient().streamingOps.queryKeys(0, testStoreName, queryKeys.iterator());
    // key 0
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(0).get(), entry.getKey().get()));
    assertEquals(0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(0)), entry.getValues().get(0).getValue()));
    assertNull(entry.getException());
    // key 1
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(1).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // key 2
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(2).get(), entry.getKey().get()));
    assertEquals(0, entry.getValues().size());
    assertNull(entry.getException());
    // key 3
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(3).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // key 4
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(4).get(), entry.getKey().get()));
    assertEquals(0, ByteUtils.compare(belongToAndInsideServer0.get(queryKeys.get(4)), entry.getValues().get(0).getValue()));
    assertNull(entry.getException());
    // key 5
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(5).get(), entry.getKey().get()));
    assertEquals(0, entry.getValues().size());
    assertNull(entry.getException());
    // key 6
    entry = results.next();
    assertEquals(0, ByteUtils.compare(queryKeys.get(6).get(), entry.getKey().get()));
    assertTrue("There should be InvalidMetadataException exception", entry.getException() instanceof InvalidMetadataException);
    // no more keys
    assertFalse(results.hasNext());
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) Node(voldemort.cluster.Node) InvalidMetadataException(voldemort.store.InvalidMetadataException) ArrayList(java.util.ArrayList) QueryKeyResult(voldemort.client.protocol.admin.QueryKeyResult) RoutingStrategy(voldemort.routing.RoutingStrategy) ByteArray(voldemort.utils.ByteArray) List(java.util.List) ArrayList(java.util.ArrayList) Test(org.junit.Test)

Example 5 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AbstractConsistencyFixer method doConsistencyFix.

public Status doConsistencyFix() {
    // Initialization.
    byte[] keyInBytes;
    List<Integer> nodeIdList = null;
    int masterPartitionId = -1;
    try {
        keyInBytes = ByteUtils.fromHexString(badKey.getKeyInHexFormat());
        masterPartitionId = this.storeInstance.getMasterPartitionId(keyInBytes);
        nodeIdList = this.storeInstance.getReplicationNodeList(masterPartitionId);
    } catch (Exception exception) {
        logger.info("Aborting fixKey due to bad init.");
        if (logger.isDebugEnabled()) {
            exception.printStackTrace();
        }
        return Status.BAD_INIT;
    }
    ByteArray keyAsByteArray = new ByteArray(keyInBytes);
    // Do the reads
    Map<Integer, QueryKeyResult> nodeIdToKeyValues = doReads(nodeIdList, keyInBytes, badKey.getKeyInHexFormat());
    // Process read replies (i.e., nodeIdToKeyValues)
    ProcessReadRepliesResult result = processReadReplies(nodeIdList, keyAsByteArray, badKey.getKeyInHexFormat(), nodeIdToKeyValues);
    if (result.status != Status.SUCCESS) {
        return result.status;
    }
    // Resolve conflicts indicated in nodeValues
    List<NodeValue<ByteArray, byte[]>> toReadRepair = resolveReadConflicts(result.nodeValues);
    if (logger.isTraceEnabled()) {
        if (toReadRepair.size() == 0) {
            logger.trace("Nothing to repair");
        }
        for (NodeValue<ByteArray, byte[]> nodeValue : toReadRepair) {
            logger.trace(nodeValue.getNodeId() + " --- " + nodeValue.getKey().toString());
        }
    }
    // Do the repairs
    Status status = doRepairPut(toReadRepair);
    // return status of last operation (success or otherwise)
    return status;
}
Also used : Status(voldemort.utils.ConsistencyFix.Status) NodeValue(voldemort.store.routed.NodeValue) VoldemortException(voldemort.VoldemortException) QueryKeyResult(voldemort.client.protocol.admin.QueryKeyResult)

Aggregations

QueryKeyResult (voldemort.client.protocol.admin.QueryKeyResult)9 Versioned (voldemort.versioning.Versioned)8 VoldemortException (voldemort.VoldemortException)5 ByteArray (voldemort.utils.ByteArray)5 ArrayList (java.util.ArrayList)4 HashMap (java.util.HashMap)3 List (java.util.List)2 Node (voldemort.cluster.Node)2 InvalidMetadataException (voldemort.store.InvalidMetadataException)2 NodeValue (voldemort.store.routed.NodeValue)2 StoreNotFoundException (com.sleepycat.persist.StoreNotFoundException)1 BufferedWriter (java.io.BufferedWriter)1 IOException (java.io.IOException)1 OutputStreamWriter (java.io.OutputStreamWriter)1 StringReader (java.io.StringReader)1 StringWriter (java.io.StringWriter)1 AbstractMap (java.util.AbstractMap)1 Date (java.util.Date)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1