Examples with DefaultSerializerFactory voldemort.serialization.DefaultSerializerFactory used on opensource projects

Example 11 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class StreamingClientTest method verifyKeysExist.

/*
     * Checks if each node has the keys it is reponsible for returns false
     * otherwise
     */
public boolean verifyKeysExist(int nodeIdToVerifyOn) {
    RoutingStrategyFactory factory = new RoutingStrategyFactory();
    RoutingStrategy storeRoutingStrategy = factory.updateRoutingStrategy(storeDef, adminClient.getAdminClientCluster());
    HashMap<Integer, ArrayList<String>> expectedNodeIdToKeys;
    expectedNodeIdToKeys = new HashMap();
    Collection<Node> nodesInCluster = adminClient.getAdminClientCluster().getNodes();
    for (Node node : nodesInCluster) {
        ArrayList<String> keysForNode = new ArrayList();
        expectedNodeIdToKeys.put(node.getId(), keysForNode);
    }
    for (int i = 0; i < NUM_KEYS_1; i++) {
        String key = i + "";
        String value = key;
        List<Node> nodeList = storeRoutingStrategy.routeRequest(key.getBytes());
        for (Node node : nodeList) {
            ArrayList<String> keysForNode = expectedNodeIdToKeys.get(node.getId());
            keysForNode.add(key);
        }
    }
    ArrayList<String> fetchedKeysForNode = new ArrayList();
    for (Node node : nodesInCluster) {
        List<Integer> partitionIdList = Lists.newArrayList();
        partitionIdList.addAll(node.getPartitionIds());
        Iterator<ByteArray> keyIteratorRef = null;
        keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(node.getId(), TEST_STORE_NAME, partitionIdList, null, false);
        final SerializerDefinition serializerDef = storeDef.getKeySerializer();
        final SerializerFactory serializerFactory = new DefaultSerializerFactory();
        @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
        final CompressionStrategy keysCompressionStrategy;
        if (serializerDef != null && serializerDef.hasCompression()) {
            keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
        } else {
            keysCompressionStrategy = null;
        }
        final Iterator<ByteArray> keyIterator = keyIteratorRef;
        while (keyIterator.hasNext()) {
            byte[] keyBytes = keyIterator.next().get();
            try {
                Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
                fetchedKeysForNode.add((String) keyObject);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    ArrayList<String> keysForNode = expectedNodeIdToKeys.get(nodeIdToVerifyOn);
    if (!fetchedKeysForNode.containsAll(keysForNode))
        return false;
    else
        return true;
}

Example 12 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class ViewStorageConfiguration method getStore.

public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    String name = storeDef.getName();
    StoreDefinition def = StoreUtils.getStoreDef(storeDefs, name);
    String targetName = def.getViewTargetStoreName();
    StoreDefinition targetDef = StoreUtils.getStoreDef(storeDefs, targetName);
    StorageEngine<ByteArray, byte[], byte[]> target = storeRepo.getStorageEngine(targetName);
    if (target == null)
        throw new VoldemortException("View \"" + name + "\" has a target store \"" + targetName + "\" which does not exist.");
    String factoryName = def.getSerializerFactory();
    SerializerFactory factory;
    if (factoryName == null)
        factory = new DefaultSerializerFactory();
    else
        factory = loadSerializerFactory(factoryName);
    CompressionStrategy valueCompressionStrategy = null;
    if (targetDef.getValueSerializer().hasCompression()) {
        valueCompressionStrategy = new CompressionStrategyFactory().get(targetDef.getValueSerializer().getCompression());
    }
    View<?, ?, ?, ?> view = loadTransformation(def.getValueTransformation());
    return new ViewStorageEngine(name, target, factory.getSerializer(def.getValueSerializer()), def.getTransformsSerializer() != null ? factory.getSerializer(def.getTransformsSerializer()) : null, factory.getSerializer(targetDef.getKeySerializer()), factory.getSerializer(targetDef.getValueSerializer()), valueCompressionStrategy, view);
}

Example 13 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class MockStoreClientFactory method getRawStore.

private <K1, V1, T1> Store<K1, V1, T1> getRawStore(String storeName) {
    List<StoreDefinition> storeDefs = storeMapper.readStoreList(new StringReader(storesXml));
    StoreDefinition storeDef = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        storeDef = d;
    if (storeDef == null)
        throw new BootstrapFailureException("Unknown store '" + storeName + "'.");
    DefaultSerializerFactory serializerFactory = new DefaultSerializerFactory();
    Serializer<K1> keySerializer = (Serializer<K1>) serializerFactory.getSerializer(storeDef.getKeySerializer());
    Serializer<V1> valueSerializer = (Serializer<V1>) serializerFactory.getSerializer(storeDef.getValueSerializer());
    Serializer<T1> transformsSerializer = null;
    if (storeDef.isView())
        transformsSerializer = (Serializer<T1>) serializerFactory.getSerializer(storeDef.getTransformsSerializer());
    // Add inconsistency resolving decorator, using their inconsistency
    // resolver (if they gave us one)
    InconsistencyResolver<Versioned<V1>> secondaryResolver = new TimeBasedInconsistencyResolver();
    StorageEngine engine;
    if (storeDef.isView())
        engine = new InMemoryStorageEngine(storeDef.getViewTargetStoreName());
    else
        engine = new InMemoryStorageEngine(storeDef.getName());
    if (storeDef.isView()) {
        // instantiate view
        String targetName = storeDef.getViewTargetStoreName();
        StoreDefinition targetDef = StoreUtils.getStoreDef(storeDefs, targetName);
        engine = new ViewStorageEngine(storeName, engine, this.viewValueSerializer != null ? this.viewValueSerializer : serializerFactory.getSerializer(storeDef.getValueSerializer()), this.transformsSerializer != null ? this.transformsSerializer : serializerFactory.getSerializer(storeDef.getTransformsSerializer()), this.keySerializer != null ? this.keySerializer : serializerFactory.getSerializer(targetDef.getKeySerializer()), this.valueSerializer != null ? this.valueSerializer : serializerFactory.getSerializer(targetDef.getValueSerializer()), null, ViewStorageConfiguration.loadTransformation(storeDef.getValueTransformation()));
    }
    Store store = new VersionIncrementingStore(engine, nodeId, time);
    store = new SerializingStore(store, this.keySerializer != null ? this.keySerializer : keySerializer, this.valueSerializer != null ? this.valueSerializer : valueSerializer, this.transformsSerializer != null ? this.transformsSerializer : transformsSerializer);
    Store<K1, V1, T1> consistentStore = new InconsistencyResolvingStore<K1, V1, T1>(store, new ChainedResolver<Versioned<V1>>(new VectorClockInconsistencyResolver(), secondaryResolver));
    return consistentStore;
}

Example 14 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class VoldemortAdminTool method executeFetchKeys.

private static void executeFetchKeys(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException {
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    Map<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap();
    for (StoreDefinition storeDefinition : storeDefinitionList) {
        storeDefinitionMap.put(storeDefinition.getName(), storeDefinition);
    }
    File directory = null;
    if (outputDir != null) {
        directory = new File(outputDir);
        if (!(directory.exists() || directory.mkdir())) {
            Utils.croak("Can't find or create directory " + outputDir);
        }
    }
    List<String> stores = storeNames;
    if (stores == null) {
        stores = Lists.newArrayList();
        stores.addAll(storeDefinitionMap.keySet());
    } else {
        // add system stores to the map so they can be fetched when
        // specified explicitly
        storeDefinitionMap.putAll(getSystemStoreDefs());
    }
    // Pick up all the partitions
    if (partitionIdList == null) {
        partitionIdList = Lists.newArrayList();
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            partitionIdList.addAll(node.getPartitionIds());
        }
    }
    StoreDefinition storeDefinition = null;
    for (String store : stores) {
        storeDefinition = storeDefinitionMap.get(store);
        if (null == storeDefinition) {
            System.out.println("No store found under the name \'" + store + "\'");
            continue;
        }
        Iterator<ByteArray> keyIteratorRef = null;
        if (fetchOrphaned) {
            System.out.println("Fetching orphaned keys  of " + store);
            keyIteratorRef = adminClient.bulkFetchOps.fetchOrphanedKeys(nodeId, store);
        } else {
            System.out.println("Fetching keys in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store);
            keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(nodeId, store, partitionIdList, null, false);
        }
        File outputFile = null;
        if (directory != null) {
            outputFile = new File(directory, store + ".keys");
        }
        final Iterator<ByteArray> keyIterator = keyIteratorRef;
        if (useAscii) {
            final SerializerDefinition serializerDef = storeDefinition.getKeySerializer();
            final SerializerFactory serializerFactory = new DefaultSerializerFactory();
            @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
            final CompressionStrategy keysCompressionStrategy;
            if (serializerDef != null && serializerDef.hasCompression()) {
                keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
            } else {
                keysCompressionStrategy = null;
            }
            writeAscii(outputFile, new Writable() {

                @Override
                public void writeTo(BufferedWriter out) throws IOException {
                    while (keyIterator.hasNext()) {
                        final JsonGenerator generator = new JsonFactory(new ObjectMapper()).createJsonGenerator(out);
                        byte[] keyBytes = keyIterator.next().get();
                        Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
                        if (keyObject instanceof GenericRecord) {
                            out.write(keyObject.toString());
                        } else {
                            generator.writeObject(keyObject);
                        }
                        out.write('\n');
                    }
                }
            });
        } else {
            writeBinary(outputFile, new Printable() {

                @Override
                public void printTo(DataOutputStream out) throws IOException {
                    while (keyIterator.hasNext()) {
                        byte[] keyBytes = keyIterator.next().get();
                        out.writeChars(ByteUtils.toHexString(keyBytes) + "\n");
                    }
                }
            });
        }
        if (outputFile != null)
            System.out.println("Fetched keys from " + store + " to " + outputFile);
    }
}

Example 15 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class VoldemortAdminTool method executeQueryKey.

private static void executeQueryKey(final Integer nodeId, AdminClient adminClient, List<String> storeNames, String keyString, String keyFormat) throws IOException {
    // decide queryingNode(s) for Key
    List<Integer> queryingNodes = new ArrayList<Integer>();
    if (nodeId < 0) {
        // means all nodes
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            queryingNodes.add(node.getId());
        }
    } else {
        queryingNodes.add(nodeId);
    }
    // get basic info
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    Map<String, StoreDefinition> storeDefinitions = new HashMap<String, StoreDefinition>();
    for (StoreDefinition storeDef : storeDefinitionList) {
        storeDefinitions.put(storeDef.getName(), storeDef);
    }
    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(System.out));
    // iterate through stores
    for (final String storeName : storeNames) {
        // store definition
        StoreDefinition storeDefinition = storeDefinitions.get(storeName);
        if (storeDefinition == null) {
            throw new StoreNotFoundException("Store " + storeName + " not found");
        }
        out.write("STORE_NAME: " + storeDefinition.getName() + "\n");
        // k-v serializer
        final SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
        final SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
        SerializerFactory serializerFactory = new DefaultSerializerFactory();
        @SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
        @SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);
        // compression strategy
        final CompressionStrategy keyCompressionStrategy;
        final CompressionStrategy valueCompressionStrategy;
        if (keySerializerDef != null && keySerializerDef.hasCompression()) {
            keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
        } else {
            keyCompressionStrategy = null;
        }
        if (valueSerializerDef != null && valueSerializerDef.hasCompression()) {
            valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
        } else {
            valueCompressionStrategy = null;
        }
        if (keyCompressionStrategy == null) {
            out.write("KEY_COMPRESSION_STRATEGY: None\n");
        } else {
            out.write("KEY_COMPRESSION_STRATEGY: " + keyCompressionStrategy.getType() + "\n");
        }
        out.write("KEY_SERIALIZER_NAME: " + keySerializerDef.getName() + "\n");
        for (Map.Entry<Integer, String> entry : keySerializerDef.getAllSchemaInfoVersions().entrySet()) {
            out.write(String.format("KEY_SCHEMA VERSION=%d\n", entry.getKey()));
            out.write("====================================\n");
            out.write(entry.getValue());
            out.write("\n====================================\n");
        }
        out.write("\n");
        if (valueCompressionStrategy == null) {
            out.write("VALUE_COMPRESSION_STRATEGY: None\n");
        } else {
            out.write("VALUE_COMPRESSION_STRATEGY: " + valueCompressionStrategy.getType() + "\n");
        }
        out.write("VALUE_SERIALIZER_NAME: " + valueSerializerDef.getName() + "\n");
        for (Map.Entry<Integer, String> entry : valueSerializerDef.getAllSchemaInfoVersions().entrySet()) {
            out.write(String.format("VALUE_SCHEMA %d\n", entry.getKey()));
            out.write("====================================\n");
            out.write(entry.getValue());
            out.write("\n====================================\n");
        }
        out.write("\n");
        // although the streamingOps support multiple keys, we only query
        // one key here
        ByteArray key;
        try {
            if (keyFormat.equals("readable")) {
                Object keyObject;
                String keySerializerName = keySerializerDef.getName();
                if (isAvroSchema(keySerializerName)) {
                    Schema keySchema = Schema.parse(keySerializerDef.getCurrentSchemaInfo());
                    JsonDecoder decoder = new JsonDecoder(keySchema, keyString);
                    GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(keySchema);
                    keyObject = datumReader.read(null, decoder);
                } else if (keySerializerName.equals(DefaultSerializerFactory.JSON_SERIALIZER_TYPE_NAME)) {
                    JsonReader jsonReader = new JsonReader(new StringReader(keyString));
                    keyObject = jsonReader.read();
                } else {
                    keyObject = keyString;
                }
                key = new ByteArray(keySerializer.toBytes(keyObject));
            } else {
                key = new ByteArray(ByteUtils.fromHexString(keyString));
            }
        } catch (SerializationException se) {
            System.err.println("Error serializing key " + keyString);
            System.err.println("If this is a JSON key, you need to include escaped quotation marks in the command line if it is a string");
            se.printStackTrace();
            return;
        } catch (DecoderException de) {
            System.err.println("Error decoding key " + keyString);
            de.printStackTrace();
            return;
        } catch (IOException io) {
            System.err.println("Error parsing avro string " + keyString);
            io.printStackTrace();
            return;
        }
        boolean printedKey = false;
        // A Map<> could have been used instead of List<Entry<>> if
        // Versioned supported correct hash codes. Read the comment in
        // Versioned about the issue
        List<Entry<List<Versioned<byte[]>>, List<Integer>>> nodeValues = new ArrayList<Entry<List<Versioned<byte[]>>, List<Integer>>>();
        for (final Integer queryNodeId : queryingNodes) {
            Iterator<QueryKeyResult> iterator;
            iterator = adminClient.streamingOps.queryKeys(queryNodeId, storeName, Arrays.asList(key).iterator());
            final StringWriter stringWriter = new StringWriter();
            QueryKeyResult queryKeyResult = iterator.next();
            if (!printedKey) {
                // de-serialize and write key
                byte[] keyBytes = queryKeyResult.getKey().get();
                Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes));
                writeVoldKeyOrValueInternal(keyBytes, keySerializer, keyCompressionStrategy, "KEY", out);
                printedKey = true;
            }
            // iterate through, de-serialize and write values
            if (queryKeyResult.hasValues() && queryKeyResult.getValues().size() > 0) {
                int elementId = -1;
                for (int i = 0; i < nodeValues.size(); i++) {
                    if (Objects.equal(nodeValues.get(i).getKey(), queryKeyResult.getValues())) {
                        elementId = i;
                        break;
                    }
                }
                if (elementId == -1) {
                    ArrayList<Integer> nodes = new ArrayList<Integer>();
                    nodes.add(queryNodeId);
                    nodeValues.add(new AbstractMap.SimpleEntry<List<Versioned<byte[]>>, List<Integer>>(queryKeyResult.getValues(), nodes));
                } else {
                    nodeValues.get(elementId).getValue().add(queryNodeId);
                }
                out.write(String.format("\nQueried node %d on store %s\n", queryNodeId, storeName));
                int versionCount = 0;
                if (queryKeyResult.getValues().size() > 1) {
                    out.write("VALUE " + versionCount + "\n");
                }
                for (Versioned<byte[]> versioned : queryKeyResult.getValues()) {
                    // write version
                    VectorClock version = (VectorClock) versioned.getVersion();
                    out.write("VECTOR_CLOCK_BYTE: " + ByteUtils.toHexString(version.toBytes()) + "\n");
                    out.write("VECTOR_CLOCK_TEXT: " + version.toString() + '[' + new Date(version.getTimestamp()).toString() + "]\n");
                    // write value
                    byte[] valueBytes = versioned.getValue();
                    writeVoldKeyOrValueInternal(valueBytes, valueSerializer, valueCompressionStrategy, "VALUE", out);
                    versionCount++;
                }
            } else // exception.
            if (queryKeyResult.hasException()) {
                boolean isInvalidMetadataException = queryKeyResult.getException() instanceof InvalidMetadataException;
                // you are querying only a single node.
                if (!isInvalidMetadataException || queryingNodes.size() == 1) {
                    out.write(String.format("\nNode %d on store %s returned exception\n", queryNodeId, storeName));
                    out.write(queryKeyResult.getException().toString());
                    out.write("\n====================================\n");
                }
            } else {
                if (queryingNodes.size() == 1) {
                    out.write(String.format("\nNode %d on store %s returned NULL\n", queryNodeId, storeName));
                    out.write("\n====================================\n");
                }
            }
            out.flush();
        }
        out.write("\n====================================\n");
        for (Map.Entry<List<Versioned<byte[]>>, List<Integer>> nodeValue : nodeValues) {
            out.write("Nodes with same Value " + Arrays.toString(nodeValue.getValue().toArray()));
            out.write("\n====================================\n");
        }
        if (nodeValues.size() > 1) {
            out.write("\n*** Multiple (" + nodeValues.size() + ") versions of key/value exist for the key ***\n");
        }
        out.flush();
    }
}