Examples with QueryKeyResult voldemort.client.protocol.admin.QueryKeyResult used on opensource projects

Example 6 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class VoldemortAdminTool method executeQueryKey.

private static void executeQueryKey(final Integer nodeId, AdminClient adminClient, List<String> storeNames, String keyString, String keyFormat) throws IOException {
    // decide queryingNode(s) for Key
    List<Integer> queryingNodes = new ArrayList<Integer>();
    if (nodeId < 0) {
        // means all nodes
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            queryingNodes.add(node.getId());
        }
    } else {
        queryingNodes.add(nodeId);
    }
    // get basic info
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    Map<String, StoreDefinition> storeDefinitions = new HashMap<String, StoreDefinition>();
    for (StoreDefinition storeDef : storeDefinitionList) {
        storeDefinitions.put(storeDef.getName(), storeDef);
    }
    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(System.out));
    // iterate through stores
    for (final String storeName : storeNames) {
        // store definition
        StoreDefinition storeDefinition = storeDefinitions.get(storeName);
        if (storeDefinition == null) {
            throw new StoreNotFoundException("Store " + storeName + " not found");
        }
        out.write("STORE_NAME: " + storeDefinition.getName() + "\n");
        // k-v serializer
        final SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
        final SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
        SerializerFactory serializerFactory = new DefaultSerializerFactory();
        @SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
        @SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);
        // compression strategy
        final CompressionStrategy keyCompressionStrategy;
        final CompressionStrategy valueCompressionStrategy;
        if (keySerializerDef != null && keySerializerDef.hasCompression()) {
            keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
        } else {
            keyCompressionStrategy = null;
        }
        if (valueSerializerDef != null && valueSerializerDef.hasCompression()) {
            valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
        } else {
            valueCompressionStrategy = null;
        }
        if (keyCompressionStrategy == null) {
            out.write("KEY_COMPRESSION_STRATEGY: None\n");
        } else {
            out.write("KEY_COMPRESSION_STRATEGY: " + keyCompressionStrategy.getType() + "\n");
        }
        out.write("KEY_SERIALIZER_NAME: " + keySerializerDef.getName() + "\n");
        for (Map.Entry<Integer, String> entry : keySerializerDef.getAllSchemaInfoVersions().entrySet()) {
            out.write(String.format("KEY_SCHEMA VERSION=%d\n", entry.getKey()));
            out.write("====================================\n");
            out.write(entry.getValue());
            out.write("\n====================================\n");
        }
        out.write("\n");
        if (valueCompressionStrategy == null) {
            out.write("VALUE_COMPRESSION_STRATEGY: None\n");
        } else {
            out.write("VALUE_COMPRESSION_STRATEGY: " + valueCompressionStrategy.getType() + "\n");
        }
        out.write("VALUE_SERIALIZER_NAME: " + valueSerializerDef.getName() + "\n");
        for (Map.Entry<Integer, String> entry : valueSerializerDef.getAllSchemaInfoVersions().entrySet()) {
            out.write(String.format("VALUE_SCHEMA %d\n", entry.getKey()));
            out.write("====================================\n");
            out.write(entry.getValue());
            out.write("\n====================================\n");
        }
        out.write("\n");
        // although the streamingOps support multiple keys, we only query
        // one key here
        ByteArray key;
        try {
            if (keyFormat.equals("readable")) {
                Object keyObject;
                String keySerializerName = keySerializerDef.getName();
                if (isAvroSchema(keySerializerName)) {
                    Schema keySchema = Schema.parse(keySerializerDef.getCurrentSchemaInfo());
                    JsonDecoder decoder = new JsonDecoder(keySchema, keyString);
                    GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(keySchema);
                    keyObject = datumReader.read(null, decoder);
                } else if (keySerializerName.equals(DefaultSerializerFactory.JSON_SERIALIZER_TYPE_NAME)) {
                    JsonReader jsonReader = new JsonReader(new StringReader(keyString));
                    keyObject = jsonReader.read();
                } else {
                    keyObject = keyString;
                }
                key = new ByteArray(keySerializer.toBytes(keyObject));
            } else {
                key = new ByteArray(ByteUtils.fromHexString(keyString));
            }
        } catch (SerializationException se) {
            System.err.println("Error serializing key " + keyString);
            System.err.println("If this is a JSON key, you need to include escaped quotation marks in the command line if it is a string");
            se.printStackTrace();
            return;
        } catch (DecoderException de) {
            System.err.println("Error decoding key " + keyString);
            de.printStackTrace();
            return;
        } catch (IOException io) {
            System.err.println("Error parsing avro string " + keyString);
            io.printStackTrace();
            return;
        }
        boolean printedKey = false;
        // A Map<> could have been used instead of List<Entry<>> if
        // Versioned supported correct hash codes. Read the comment in
        // Versioned about the issue
        List<Entry<List<Versioned<byte[]>>, List<Integer>>> nodeValues = new ArrayList<Entry<List<Versioned<byte[]>>, List<Integer>>>();
        for (final Integer queryNodeId : queryingNodes) {
            Iterator<QueryKeyResult> iterator;
            iterator = adminClient.streamingOps.queryKeys(queryNodeId, storeName, Arrays.asList(key).iterator());
            final StringWriter stringWriter = new StringWriter();
            QueryKeyResult queryKeyResult = iterator.next();
            if (!printedKey) {
                // de-serialize and write key
                byte[] keyBytes = queryKeyResult.getKey().get();
                Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes));
                writeVoldKeyOrValueInternal(keyBytes, keySerializer, keyCompressionStrategy, "KEY", out);
                printedKey = true;
            }
            // iterate through, de-serialize and write values
            if (queryKeyResult.hasValues() && queryKeyResult.getValues().size() > 0) {
                int elementId = -1;
                for (int i = 0; i < nodeValues.size(); i++) {
                    if (Objects.equal(nodeValues.get(i).getKey(), queryKeyResult.getValues())) {
                        elementId = i;
                        break;
                    }
                }
                if (elementId == -1) {
                    ArrayList<Integer> nodes = new ArrayList<Integer>();
                    nodes.add(queryNodeId);
                    nodeValues.add(new AbstractMap.SimpleEntry<List<Versioned<byte[]>>, List<Integer>>(queryKeyResult.getValues(), nodes));
                } else {
                    nodeValues.get(elementId).getValue().add(queryNodeId);
                }
                out.write(String.format("\nQueried node %d on store %s\n", queryNodeId, storeName));
                int versionCount = 0;
                if (queryKeyResult.getValues().size() > 1) {
                    out.write("VALUE " + versionCount + "\n");
                }
                for (Versioned<byte[]> versioned : queryKeyResult.getValues()) {
                    // write version
                    VectorClock version = (VectorClock) versioned.getVersion();
                    out.write("VECTOR_CLOCK_BYTE: " + ByteUtils.toHexString(version.toBytes()) + "\n");
                    out.write("VECTOR_CLOCK_TEXT: " + version.toString() + '[' + new Date(version.getTimestamp()).toString() + "]\n");
                    // write value
                    byte[] valueBytes = versioned.getValue();
                    writeVoldKeyOrValueInternal(valueBytes, valueSerializer, valueCompressionStrategy, "VALUE", out);
                    versionCount++;
                }
            } else // exception.
            if (queryKeyResult.hasException()) {
                boolean isInvalidMetadataException = queryKeyResult.getException() instanceof InvalidMetadataException;
                // you are querying only a single node.
                if (!isInvalidMetadataException || queryingNodes.size() == 1) {
                    out.write(String.format("\nNode %d on store %s returned exception\n", queryNodeId, storeName));
                    out.write(queryKeyResult.getException().toString());
                    out.write("\n====================================\n");
                }
            } else {
                if (queryingNodes.size() == 1) {
                    out.write(String.format("\nNode %d on store %s returned NULL\n", queryNodeId, storeName));
                    out.write("\n====================================\n");
                }
            }
            out.flush();
        }
        out.write("\n====================================\n");
        for (Map.Entry<List<Versioned<byte[]>>, List<Integer>> nodeValue : nodeValues) {
            out.write("Nodes with same Value " + Arrays.toString(nodeValue.getValue().toArray()));
            out.write("\n====================================\n");
        }
        if (nodeValues.size() > 1) {
            out.write("\n*** Multiple (" + nodeValues.size() + ") versions of key/value exist for the key ***\n");
        }
        out.flush();
    }
}

Example 7 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AbstractRebalanceTest method checkForKeyNonExistence.

/**
     * REFACTOR: these should belong AdminClient so existence checks can be done
     * easily across the board
     * 
     * @param admin
     * @param serverId
     * @param store
     * @param keyList
     */
protected void checkForKeyNonExistence(AdminClient admin, int serverId, String store, List<ByteArray> keyList) {
    Iterator<QueryKeyResult> negativeTestResultsItr = admin.streamingOps.queryKeys(serverId, store, keyList.iterator());
    while (negativeTestResultsItr.hasNext()) {
        QueryKeyResult item = negativeTestResultsItr.next();
        ByteArray key = item.getKey();
        List<Versioned<byte[]>> vals = item.getValues();
        Exception e = item.getException();
        assertEquals("Error fetching key " + key, null, e);
        assertEquals("Value " + vals + "found for key " + key, true, vals == null || vals.size() == 0);
    }
}

Example 8 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AbstractConsistencyFixer method doConsistencyFix.

public Status doConsistencyFix() {
    // Initialization.
    byte[] keyInBytes;
    List<Integer> nodeIdList = null;
    int masterPartitionId = -1;
    try {
        keyInBytes = ByteUtils.fromHexString(badKey.getKeyInHexFormat());
        masterPartitionId = this.storeInstance.getMasterPartitionId(keyInBytes);
        nodeIdList = this.storeInstance.getReplicationNodeList(masterPartitionId);
    } catch (Exception exception) {
        logger.info("Aborting fixKey due to bad init.");
        if (logger.isDebugEnabled()) {
            exception.printStackTrace();
        }
        return Status.BAD_INIT;
    }
    ByteArray keyAsByteArray = new ByteArray(keyInBytes);
    // Do the reads
    Map<Integer, QueryKeyResult> nodeIdToKeyValues = doReads(nodeIdList, keyInBytes, badKey.getKeyInHexFormat());
    // Process read replies (i.e., nodeIdToKeyValues)
    ProcessReadRepliesResult result = processReadReplies(nodeIdList, keyAsByteArray, badKey.getKeyInHexFormat(), nodeIdToKeyValues);
    if (result.status != Status.SUCCESS) {
        return result.status;
    }
    // Resolve conflicts indicated in nodeValues
    List<NodeValue<ByteArray, byte[]>> toReadRepair = resolveReadConflicts(result.nodeValues);
    if (logger.isTraceEnabled()) {
        if (toReadRepair.size() == 0) {
            logger.trace("Nothing to repair");
        }
        for (NodeValue<ByteArray, byte[]> nodeValue : toReadRepair) {
            logger.trace(nodeValue.getNodeId() + " --- " + nodeValue.getKey().toString());
        }
    }
    // Do the repairs
    Status status = doRepairPut(toReadRepair);
    // return status of last operation (success or otherwise)
    return status;
}

Example 9 with QueryKeyResult

use of voldemort.client.protocol.admin.QueryKeyResult in project voldemort by voldemort.

the class AbstractConsistencyFixer method processReadReplies.

/**
     * 
     * @param nodeIdList
     * @param keyAsByteArray
     * @param keyInHexFormat
     * @param nodeIdToKeyValues
     * @param nodeValues Effectively the output of this method. Must pass in a
     *        non-null object to be populated by this method.
     * @return
     */
private ProcessReadRepliesResult processReadReplies(final List<Integer> nodeIdList, final ByteArray keyAsByteArray, final String keyInHexFormat, final Map<Integer, QueryKeyResult> nodeIdToKeyValues) {
    List<NodeValue<ByteArray, byte[]>> nodeValues = new ArrayList<NodeValue<ByteArray, byte[]>>();
    boolean exceptionsEncountered = false;
    for (int nodeId : nodeIdList) {
        QueryKeyResult keyValue;
        if (nodeIdToKeyValues.containsKey(nodeId)) {
            keyValue = nodeIdToKeyValues.get(nodeId);
            if (keyValue.hasException()) {
                logger.debug("Exception encountered while fetching key " + keyInHexFormat + " from node with nodeId " + nodeId + " : " + keyValue.getException().getMessage());
                exceptionsEncountered = true;
            } else {
                if (keyValue.getValues().isEmpty()) {
                    Versioned<byte[]> versioned = new Versioned<byte[]>(null);
                    nodeValues.add(new NodeValue<ByteArray, byte[]>(nodeId, keyValue.getKey(), versioned));
                } else {
                    for (Versioned<byte[]> value : keyValue.getValues()) {
                        nodeValues.add(new NodeValue<ByteArray, byte[]>(nodeId, keyValue.getKey(), value));
                    }
                }
            }
        } else {
            logger.debug("No key-value returned from node with id:" + nodeId);
            Versioned<byte[]> versioned = new Versioned<byte[]>(null);
            nodeValues.add(new NodeValue<ByteArray, byte[]>(nodeId, keyAsByteArray, versioned));
        }
    }
    if (exceptionsEncountered) {
        logger.info("Aborting fixKey because exceptions were encountered when fetching key-values.");
        return new ProcessReadRepliesResult(Status.FETCH_EXCEPTION);
    }
    if (logger.isDebugEnabled()) {
        for (NodeValue<ByteArray, byte[]> nkv : nodeValues) {
            logger.debug("\tRead NodeKeyValue : " + ByteUtils.toHexString(nkv.getKey().get()) + " on node with id " + nkv.getNodeId() + " for version " + nkv.getVersion());
        }
    }
    return new ProcessReadRepliesResult(nodeValues);
}