Examples with Pair voldemort.utils.Pair used on opensource projects

Example 26 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class VoldemortAvroClientShell method getLatestKeyValueSchema.

private static Pair<Schema, Schema> getLatestKeyValueSchema(String url, String storeName) {
    AdminClient adminClient = null;
    try {
        adminClient = new AdminClient(url);
        List<StoreDefinition> storeDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList().getValue();
        for (StoreDefinition storeDef : storeDefs) {
            if (storeDef.getName().equals(storeName)) {
                Schema keySchema = Schema.parse(storeDef.getKeySerializer().getCurrentSchemaInfo());
                Schema valueSchema = Schema.parse(storeDef.getValueSerializer().getCurrentSchemaInfo());
                return new Pair<Schema, Schema>(keySchema, valueSchema);
            }
        }
    } catch (Exception e) {
        System.err.println("Error while getting latest key schema " + e.getMessage());
    } finally {
        if (adminClient != null) {
            adminClient.close();
        }
    }
    return null;
}

Example 27 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class HadoopStoreBuilderTest method testHadoopBuild.

@Test
public void testHadoopBuild() throws Exception {
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp"), tempDir2 = new File(testDir, "temp2");
    File outputDir = new File(testDir, "output"), outputDir2 = new File(testDir, "output2");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < 200; i++) values.put(Integer.toString(i), Integer.toBinaryString(i));
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    for (Map.Entry<String, String> entry : values.entrySet()) contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
    FileUtils.writeStringToFile(inputFile, contents.toString());
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    // Test backwards compatibility
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir2.getAbsolutePath()), new Path(outputDir2.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    // Check if checkSum is generated in outputDir
    File nodeFile = new File(outputDir, "node-0");
    // Check if metadata file exists
    File metadataFile = new File(nodeFile, ".metadata");
    Assert.assertTrue("Metadata file should exist!", metadataFile.exists());
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata(metadataFile);
    if (saveKeys)
        Assert.assertEquals("In saveKeys mode, the metadata format should be READONLY_V2!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V2.getCode());
    else
        Assert.assertEquals("In legacy mode (saveKeys==false), the metadata format should be READONLY_V1!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V1.getCode());
    Assert.assertEquals("Checksum type should be MD5!", metadata.get(ReadOnlyStorageMetadata.CHECKSUM_TYPE), CheckSum.toString(CheckSumType.MD5));
    // Check contents of checkSum file
    byte[] md5 = Hex.decodeHex(((String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM)).toCharArray());
    byte[] checkSumBytes = CheckSumTests.calculateCheckSum(nodeFile.listFiles(), CheckSumType.MD5);
    Assert.assertEquals("Checksum is not as excepted!", 0, ByteUtils.compare(checkSumBytes, md5));
    // check if fetching works
    HdfsFetcher fetcher = new HdfsFetcher();
    // Fetch to version directory
    File versionDir = new File(storeDir, "version-0");
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    Assert.assertTrue("Version directory should exist!", versionDir.exists());
    // open store
    @SuppressWarnings("unchecked") Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, searchStrategy, new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        String key = entry.getKey();
        try {
            List<Versioned<Object>> found = store.get(key, null);
            Assert.assertEquals("Incorrect number of results", 1, found.size());
            Assert.assertEquals(entry.getValue(), found.get(0).getValue());
        } catch (VoldemortException e) {
            throw new VoldemortException("Got an exception while trying to get key '" + key + "'.", e);
        }
    }
    // also check the iterator - first key iterator...
    try {
        ClosableIterator<ByteArray> keyIterator = engine.keys();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (keyIterator.hasNext()) {
            Assert.assertTrue(values.containsKey(serializer.toObject(keyIterator.next().get())));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
    // ... and entry iterator
    try {
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
}

Example 28 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class VoldemortAdminTool method readEntriesBinary.

private static Iterator<Pair<ByteArray, Versioned<byte[]>>> readEntriesBinary(File inputDir, String storeName) throws IOException {
    File inputFile = new File(inputDir, storeName + ".entries");
    if (!inputFile.exists()) {
        throw new FileNotFoundException("File " + inputFile.getAbsolutePath() + " does not exist!");
    }
    final DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(inputFile)));
    return new AbstractIterator<Pair<ByteArray, Versioned<byte[]>>>() {

        @Override
        protected Pair<ByteArray, Versioned<byte[]>> computeNext() {
            try {
                int length = dis.readInt();
                byte[] keyBytes = new byte[length];
                ByteUtils.read(dis, keyBytes);
                length = dis.readInt();
                byte[] versionBytes = new byte[length];
                ByteUtils.read(dis, versionBytes);
                length = dis.readInt();
                byte[] valueBytes = new byte[length];
                ByteUtils.read(dis, valueBytes);
                ByteArray key = new ByteArray(keyBytes);
                VectorClock version = new VectorClock(versionBytes);
                Versioned<byte[]> value = new Versioned<byte[]>(valueBytes, version);
                return new Pair<ByteArray, Versioned<byte[]>>(key, value);
            } catch (EOFException e) {
                try {
                    dis.close();
                } catch (IOException ie) {
                    ie.printStackTrace();
                }
                return endOfData();
            } catch (IOException e) {
                try {
                    dis.close();
                } catch (IOException ie) {
                    ie.printStackTrace();
                }
                throw new VoldemortException("Error reading from input file ", e);
            }
        }
    };
}

Example 29 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class VoldemortAdminTool method executeFetchEntries.

private static void executeFetchEntries(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException {
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    HashMap<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap();
    for (StoreDefinition storeDefinition : storeDefinitionList) {
        storeDefinitionMap.put(storeDefinition.getName(), storeDefinition);
    }
    File directory = null;
    if (outputDir != null) {
        directory = new File(outputDir);
        if (!(directory.exists() || directory.mkdir())) {
            Utils.croak("Can't find or create directory " + outputDir);
        }
    }
    List<String> stores = storeNames;
    if (stores == null) {
        // when no stores specified, all user defined store will be fetched,
        // but not system stores.
        stores = Lists.newArrayList();
        stores.addAll(storeDefinitionMap.keySet());
    } else {
        // add system stores to the map so they can be fetched when
        // specified explicitly
        storeDefinitionMap.putAll(getSystemStoreDefs());
    }
    // Pick up all the partitions
    if (partitionIdList == null) {
        partitionIdList = Lists.newArrayList();
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            partitionIdList.addAll(node.getPartitionIds());
        }
    }
    StoreDefinition storeDefinition = null;
    for (String store : stores) {
        storeDefinition = storeDefinitionMap.get(store);
        if (null == storeDefinition) {
            System.out.println("No store found under the name \'" + store + "\'");
            continue;
        }
        Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIteratorRef = null;
        if (fetchOrphaned) {
            System.out.println("Fetching orphaned entries of " + store);
            entriesIteratorRef = adminClient.bulkFetchOps.fetchOrphanedEntries(nodeId, store);
        } else {
            System.out.println("Fetching entries in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store);
            entriesIteratorRef = adminClient.bulkFetchOps.fetchEntries(nodeId, store, partitionIdList, null, false);
        }
        final Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = entriesIteratorRef;
        File outputFile = null;
        if (directory != null) {
            outputFile = new File(directory, store + ".entries");
        }
        if (useAscii) {
            // k-v serializer
            SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
            SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
            SerializerFactory serializerFactory = new DefaultSerializerFactory();
            @SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
            @SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);
            // compression strategy
            final CompressionStrategy keyCompressionStrategy;
            final CompressionStrategy valueCompressionStrategy;
            if (keySerializerDef != null && keySerializerDef.hasCompression()) {
                keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
            } else {
                keyCompressionStrategy = null;
            }
            if (valueSerializerDef != null && valueSerializerDef.hasCompression()) {
                valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
            } else {
                valueCompressionStrategy = null;
            }
            writeAscii(outputFile, new Writable() {

                @Override
                public void writeTo(BufferedWriter out) throws IOException {
                    while (entriesIterator.hasNext()) {
                        final JsonGenerator generator = new JsonFactory(new ObjectMapper()).createJsonGenerator(out);
                        Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next();
                        byte[] keyBytes = kvPair.getFirst().get();
                        byte[] valueBytes = kvPair.getSecond().getValue();
                        VectorClock version = (VectorClock) kvPair.getSecond().getVersion();
                        Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes));
                        Object valueObject = valueSerializer.toObject((null == valueCompressionStrategy) ? valueBytes : valueCompressionStrategy.inflate(valueBytes));
                        if (keyObject instanceof GenericRecord) {
                            out.write(keyObject.toString());
                        } else {
                            generator.writeObject(keyObject);
                        }
                        out.write(' ' + version.toString() + ' ');
                        if (valueObject instanceof GenericRecord) {
                            out.write(valueObject.toString());
                        } else {
                            generator.writeObject(valueObject);
                        }
                        out.write('\n');
                    }
                }
            });
        } else {
            writeBinary(outputFile, new Printable() {

                @Override
                public void printTo(DataOutputStream out) throws IOException {
                    while (entriesIterator.hasNext()) {
                        Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next();
                        byte[] keyBytes = kvPair.getFirst().get();
                        VectorClock clock = ((VectorClock) kvPair.getSecond().getVersion());
                        byte[] valueBytes = kvPair.getSecond().getValue();
                        out.writeChars(ByteUtils.toHexString(keyBytes));
                        out.writeChars(",");
                        out.writeChars(clock.toString());
                        out.writeChars(",");
                        out.writeChars(ByteUtils.toHexString(valueBytes));
                        out.writeChars("\n");
                    }
                }
            });
        }
        if (outputFile != null)
            System.out.println("Fetched keys from " + store + " to " + outputFile);
    }
}

Example 30 with Pair

use of voldemort.utils.Pair in project voldemort by voldemort.

the class VoldemortClientShell method evaluateCommand.

// useful as this separates the repeated prompt from the evaluation
// using no modifier as no sub-class will have access but all classes within
// package will
boolean evaluateCommand(String line, boolean printCommands) {
    try {
        if (line.toLowerCase().startsWith("put")) {
            processPut(line.substring("put".length()));
        } else if (line.toLowerCase().startsWith("getall")) {
            processGetAll(line.substring("getall".length()));
        } else if (line.toLowerCase().startsWith("getmetadata")) {
            String[] args = line.substring("getmetadata".length() + 1).split("\\s+");
            int remoteNodeId = Integer.valueOf(args[0]);
            String key = args[1];
            Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(remoteNodeId, key);
            if (versioned == null) {
                commandOutput.println("null");
            } else {
                commandOutput.println(versioned.getVersion());
                commandOutput.print(": ");
                commandOutput.println(versioned.getValue());
                commandOutput.println();
            }
        } else if (line.toLowerCase().startsWith("get")) {
            processGet(line.substring("get".length()));
        } else if (line.toLowerCase().startsWith("delete")) {
            processDelete(line.substring("delete".length()));
        } else if (line.startsWith("preflist")) {
            processPreflist(line.substring("preflist".length()));
        } else if (line.toLowerCase().startsWith("fetchkeys")) {
            String[] args = line.substring("fetchkeys".length() + 1).split("\\s+");
            int remoteNodeId = Integer.valueOf(args[0]);
            String storeName = args[1];
            List<Integer> partititionList = parseCsv(args[2]);
            Iterator<ByteArray> partitionKeys = adminClient.bulkFetchOps.fetchKeys(remoteNodeId, storeName, partititionList, null, false);
            BufferedWriter writer = null;
            try {
                if (args.length > 3) {
                    writer = new BufferedWriter(new FileWriter(new File(args[3])));
                } else
                    writer = new BufferedWriter(new OutputStreamWriter(commandOutput));
            } catch (IOException e) {
                errorStream.println("Failed to open the output stream");
                e.printStackTrace(errorStream);
            }
            if (writer != null) {
                while (partitionKeys.hasNext()) {
                    ByteArray keyByteArray = partitionKeys.next();
                    StringBuilder lineBuilder = new StringBuilder();
                    lineBuilder.append(ByteUtils.getString(keyByteArray.get(), "UTF-8"));
                    lineBuilder.append("\n");
                    writer.write(lineBuilder.toString());
                }
                writer.flush();
            }
        } else if (line.toLowerCase().startsWith("fetch")) {
            String[] args = line.substring("fetch".length() + 1).split("\\s+");
            int remoteNodeId = Integer.valueOf(args[0]);
            String storeName = args[1];
            List<Integer> partititionList = parseCsv(args[2]);
            Iterator<Pair<ByteArray, Versioned<byte[]>>> partitionEntries = adminClient.bulkFetchOps.fetchEntries(remoteNodeId, storeName, partititionList, null, false);
            BufferedWriter writer = null;
            try {
                if (args.length > 3) {
                    writer = new BufferedWriter(new FileWriter(new File(args[3])));
                } else
                    writer = new BufferedWriter(new OutputStreamWriter(commandOutput));
            } catch (IOException e) {
                errorStream.println("Failed to open the output stream");
                e.printStackTrace(errorStream);
            }
            if (writer != null) {
                while (partitionEntries.hasNext()) {
                    Pair<ByteArray, Versioned<byte[]>> pair = partitionEntries.next();
                    ByteArray keyByteArray = pair.getFirst();
                    Versioned<byte[]> versioned = pair.getSecond();
                    StringBuilder lineBuilder = new StringBuilder();
                    lineBuilder.append(ByteUtils.getString(keyByteArray.get(), "UTF-8"));
                    lineBuilder.append("\t");
                    lineBuilder.append(versioned.getVersion());
                    lineBuilder.append("\t");
                    lineBuilder.append(ByteUtils.getString(versioned.getValue(), "UTF-8"));
                    lineBuilder.append("\n");
                    writer.write(lineBuilder.toString());
                }
                writer.flush();
            }
        } else if (line.startsWith("help")) {
            commandOutput.println();
            commandOutput.println("Commands:");
            commandOutput.println(PROMPT + "put key value --- Associate the given value with the key.");
            commandOutput.println(PROMPT + "get key --- Retrieve the value associated with the key.");
            commandOutput.println(PROMPT + "getall key1 [key2...] --- Retrieve the value(s) associated with the key(s).");
            commandOutput.println(PROMPT + "delete key --- Remove all values associated with the key.");
            commandOutput.println(PROMPT + "preflist key --- Get node preference list for given key.");
            String metaKeyValues = voldemort.store.metadata.MetadataStore.METADATA_KEYS.toString();
            commandOutput.println(PROMPT + "getmetadata node_id meta_key --- Get store metadata associated " + "with meta_key from node_id. meta_key may be one of " + metaKeyValues.substring(1, metaKeyValues.length() - 1) + ".");
            commandOutput.println(PROMPT + "fetchkeys node_id store_name partitions <file_name> --- Fetch all keys " + "from given partitions (a comma separated list) of store_name on " + "node_id. Optionally, write to file_name. " + "Use getmetadata to determine appropriate values for store_name and partitions");
            commandOutput.println(PROMPT + "fetch node_id store_name partitions <file_name> --- Fetch all entries " + "from given partitions (a comma separated list) of store_name on " + "node_id. Optionally, write to file_name. " + "Use getmetadata to determine appropriate values for store_name and partitions");
            commandOutput.println(PROMPT + "help --- Print this message.");
            commandOutput.println(PROMPT + "exit --- Exit from this shell.");
            commandOutput.println();
            commandOutput.println("Avro usage:");
            commandOutput.println("For avro keys or values, ensure that the entire json string is enclosed within single quotes (').");
            commandOutput.println("Also, the field names and strings should STRICTLY be enclosed by double quotes(\")");
            commandOutput.println("eg: > put '{\"id\":1,\"name\":\"Vinoth Chandar\"}' '[{\"skill\":\"java\", \"score\":90.27, \"isendorsed\": true}]'");
        } else if (line.equals("quit") || line.equals("exit")) {
            commandOutput.println("bye.");
            System.exit(0);
        } else {
            errorStream.println("Invalid command. (Try 'help' for usage.)");
            return false;
        }
    } catch (EndOfFileException e) {
        errorStream.println("Expected additional token.");
    } catch (SerializationException e) {
        errorStream.print("Error serializing values: ");
        e.printStackTrace(errorStream);
    } catch (VoldemortException e) {
        errorStream.println("Exception thrown during operation.");
        e.printStackTrace(errorStream);
    } catch (ArrayIndexOutOfBoundsException e) {
        errorStream.println("Invalid command. (Try 'help' for usage.)");
    } catch (Exception e) {
        errorStream.println("Unexpected error:");
        e.printStackTrace(errorStream);
    }
    return true;
}