Examples with Serializer voldemort.serialization.Serializer used on opensource projects

Example 6 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class ReadOnlyStorageEngineTestInstance method create.

public static ReadOnlyStorageEngineTestInstance create(SearchStrategy strategy, File baseDir, int testSize, int numNodes, int repFactor, SerializerDefinition keySerDef, SerializerDefinition valueSerDef, ReadOnlyStorageFormat type, int[][] partitionMap) throws Exception {
    // create some test data
    Map<String, String> data = createTestData(testSize);
    JsonReader reader = makeTestDataReader(data, baseDir);
    // set up definitions for cluster and store
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < numNodes; i++) {
        List<Integer> partitions = new ArrayList<Integer>(partitionMap[i].length);
        for (int p : partitionMap[i]) {
            partitions.add(p);
        }
        nodes.add(new Node(i, "localhost", 8080 + i, 6666 + i, 7000 + i, partitions));
    }
    Cluster cluster = new Cluster("test", nodes);
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(keySerDef).setValueSerializer(valueSerDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(repFactor).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // build store files in outputDir
    File outputDir = TestUtils.createTempDir(baseDir);
    JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testSize / 5, 1, 2, 10000, false);
    storeBuilder.build(type);
    File nodeDir = TestUtils.createTempDir(baseDir);
    @SuppressWarnings("unchecked") Serializer<String> keySerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(keySerDef);
    @SuppressWarnings("unchecked") Serializer<String> valueSerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(valueSerDef);
    Serializer<String> transSerializer = new StringSerializer();
    Map<Integer, Store<String, String, String>> nodeStores = Maps.newHashMap();
    Map<Integer, ReadOnlyStorageEngine> readOnlyStores = Maps.newHashMap();
    for (int i = 0; i < numNodes; i++) {
        File currNode = new File(nodeDir, Integer.toString(i));
        currNode.mkdirs();
        currNode.deleteOnExit();
        Utils.move(new File(outputDir, "node-" + Integer.toString(i)), new File(currNode, "version-0"));
        CompressionStrategyFactory compressionStrategyFactory = new CompressionStrategyFactory();
        CompressionStrategy keyCompressionStrat = compressionStrategyFactory.get(keySerDef.getCompression());
        CompressionStrategy valueCompressionStrat = compressionStrategyFactory.get(valueSerDef.getCompression());
        ReadOnlyStorageEngine readOnlyStorageEngine = new ReadOnlyStorageEngine("test", strategy, router, i, currNode, 1);
        readOnlyStores.put(i, readOnlyStorageEngine);
        Store<ByteArray, byte[], byte[]> innerStore = new CompressingStore(readOnlyStorageEngine, keyCompressionStrat, valueCompressionStrat);
        nodeStores.put(i, SerializingStore.wrap(innerStore, keySerializer, valueSerializer, transSerializer));
    }
    return new ReadOnlyStorageEngineTestInstance(data, baseDir, readOnlyStores, nodeStores, router, keySerializer);
}

Example 7 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class AbstractStoreClientFactory method getRawStore.

@SuppressWarnings("unchecked")
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver, String customStoresXml, String clusterXmlString, FailureDetector fd) {
    logger.info("Client zone-id [" + this.routedStoreConfig.getClientZoneId() + "] Attempting to get raw store [" + storeName + "] ");
    if (logger.isDebugEnabled()) {
        for (URI uri : bootstrapUrls) {
            logger.debug("Client Bootstrap url [" + uri + "]");
        }
    }
    // Get cluster and store metadata
    String clusterXml = clusterXmlString;
    if (clusterXml == null) {
        logger.debug("Fetching cluster.xml ...");
        clusterXml = bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY, bootstrapUrls);
    }
    this.cluster = clusterMapper.readCluster(new StringReader(clusterXml), false);
    String storesXml = customStoresXml;
    if (storesXml == null) {
        String storesKey = storeName;
        if (config.isFetchAllStoresXmlInBootstrap()) {
            storesKey = MetadataStore.STORES_KEY;
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Fetching store definition for Store " + storeName + " key " + storesKey);
        }
        storesXml = bootstrapMetadataWithRetries(storesKey, bootstrapUrls);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Obtained cluster metadata xml" + clusterXml);
        logger.debug("Obtained stores  metadata xml" + storesXml);
    }
    storeDefs = storeMapper.readStoreList(new StringReader(storesXml), false);
    StoreDefinition storeDef = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        storeDef = d;
    if (storeDef == null) {
        logger.error("Bootstrap - unknown store: " + storeName);
        throw new BootstrapFailureException("Unknown store '" + storeName + "'.");
    }
    if (logger.isDebugEnabled()) {
        logger.debug(this.cluster.toString(true));
        logger.debug(storeDef.toString());
    }
    boolean repairReads = !storeDef.isView();
    // construct mapping
    Map<Integer, Store<ByteArray, byte[], byte[]>> clientMapping = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = null;
    if (storeDef.hasHintedHandoffStrategyType())
        slopStores = Maps.newHashMap();
    for (Node node : this.cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> store = getStore(storeDef.getName(), node.getHost(), getPort(node), this.requestFormatType);
        clientMapping.put(node.getId(), store);
        NonblockingStore nonblockingStore = routedStoreFactory.toNonblockingStore(store);
        nonblockingStores.put(node.getId(), nonblockingStore);
        if (slopStores != null) {
            Store<ByteArray, byte[], byte[]> rawSlopStore = getStore("slop", node.getHost(), getPort(node), this.requestFormatType);
            Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(rawSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
            slopStores.put(node.getId(), slopStore);
            nonblockingSlopStores.put(node.getId(), routedStoreFactory.toNonblockingStore(rawSlopStore));
        }
    }
    /*
         * Check if we need to retrieve a reference to the failure detector. For
         * system stores - the FD reference would be passed in.
         */
    FailureDetector failureDetectorRef = fd;
    if (failureDetectorRef == null) {
        failureDetectorRef = getFailureDetector();
    } else {
        logger.debug("Using existing failure detector.");
    }
    this.routedStoreConfig.setRepairReads(repairReads);
    Store<ByteArray, byte[], byte[]> store = routedStoreFactory.create(this.cluster, storeDef, clientMapping, nonblockingStores, slopStores, nonblockingSlopStores, failureDetectorRef, this.routedStoreConfig);
    store = new LoggingStore(store);
    if (isJmxEnabled) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.aggregateStats, this.cachedStoreStats);
        statStore.getStats().registerJmx(identifierString);
        store = statStore;
    }
    if (this.config.isEnableCompressionLayer()) {
        if (storeDef.getKeySerializer().hasCompression() || storeDef.getValueSerializer().hasCompression()) {
            store = new CompressingStore(store, getCompressionStrategy(storeDef.getKeySerializer()), getCompressionStrategy(storeDef.getValueSerializer()));
        }
    }
    /*
         * Initialize the finalstore object only once the store object itself is
         * wrapped by a StatrackingStore seems like the finalstore object is
         * redundant?
         */
    Store<K, V, T> finalStore = (Store<K, V, T>) store;
    if (this.config.isEnableSerializationLayer()) {
        Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(storeDef.getKeySerializer());
        Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(storeDef.getValueSerializer());
        if (storeDef.isView() && (storeDef.getTransformsSerializer() == null))
            throw new SerializationException("Transforms serializer must be specified with a view ");
        Serializer<T> transformsSerializer = (Serializer<T>) serializerFactory.getSerializer(storeDef.getTransformsSerializer() != null ? storeDef.getTransformsSerializer() : new SerializerDefinition("identity"));
        finalStore = SerializingStore.wrap(store, keySerializer, valueSerializer, transformsSerializer);
    }
    // resolver (if they gave us one)
    if (this.config.isEnableInconsistencyResolvingLayer()) {
        InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver() : resolver;
        finalStore = new InconsistencyResolvingStore<K, V, T>(finalStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver(), secondaryResolver));
    }
    return finalStore;
}

Example 8 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class DefaultStoreClient method getResponsibleNodes.

public List<Node> getResponsibleNodes(K key) {
    RoutingStrategy strategy = (RoutingStrategy) store.getCapability(StoreCapabilityType.ROUTING_STRATEGY);
    @SuppressWarnings("unchecked") Serializer<K> keySerializer = (Serializer<K>) store.getCapability(StoreCapabilityType.KEY_SERIALIZER);
    return strategy.routeRequest(keySerializer.toBytes(key));
}

Example 9 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class JsonTypeSerializerTest method testMapTypeKeyOrder.

public void testMapTypeKeyOrder() {
    String typeSig = "{\"name\":\"string\", \"color\":\"string\", \"age\":\"int32\"}";
    Object data = ImmutableMap.of("name", "xxx", "color", "yyy", "age", 28);
    Serializer outOfOrderSerializer = new JsonTypeSerializer(new JsonTypeDefinition(ImmutableMap.of("name", JsonTypes.STRING, "color", JsonTypes.STRING, "age", JsonTypes.INT32)));
    assertEquals("Key order in Map should not matter while serializing/deserializing", data, outOfOrderSerializer.toObject(getSerializer(typeSig).toBytes(data)));
    outOfOrderSerializer = new JsonTypeSerializer(new JsonTypeDefinition(ImmutableMap.of("color", JsonTypes.STRING, "age", JsonTypes.INT32, "name", JsonTypes.STRING)));
    assertEquals("Key order in Map should not matter while serializing/deserializing", data, outOfOrderSerializer.toObject(getSerializer(typeSig).toBytes(data)));
    outOfOrderSerializer = new JsonTypeSerializer(new JsonTypeDefinition(ImmutableMap.of("age", JsonTypes.INT32, "name", JsonTypes.STRING, "color", JsonTypes.STRING)));
    assertEquals("Key order in Map should not matter while serializing/deserializing", data, outOfOrderSerializer.toObject(getSerializer(typeSig).toBytes(data)));
    outOfOrderSerializer = new JsonTypeSerializer(new JsonTypeDefinition(ImmutableMap.of("color", JsonTypes.STRING, "name", JsonTypes.STRING, "age", JsonTypes.INT32)));
    assertEquals("Key order in Map should not matter while serializing/deserializing", data, outOfOrderSerializer.toObject(getSerializer(typeSig).toBytes(data)));
}

Example 10 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class VoldemortAdminTool method executeFetchKeys.

private static void executeFetchKeys(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException {
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    Map<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap();
    for (StoreDefinition storeDefinition : storeDefinitionList) {
        storeDefinitionMap.put(storeDefinition.getName(), storeDefinition);
    }
    File directory = null;
    if (outputDir != null) {
        directory = new File(outputDir);
        if (!(directory.exists() || directory.mkdir())) {
            Utils.croak("Can't find or create directory " + outputDir);
        }
    }
    List<String> stores = storeNames;
    if (stores == null) {
        stores = Lists.newArrayList();
        stores.addAll(storeDefinitionMap.keySet());
    } else {
        // add system stores to the map so they can be fetched when
        // specified explicitly
        storeDefinitionMap.putAll(getSystemStoreDefs());
    }
    // Pick up all the partitions
    if (partitionIdList == null) {
        partitionIdList = Lists.newArrayList();
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            partitionIdList.addAll(node.getPartitionIds());
        }
    }
    StoreDefinition storeDefinition = null;
    for (String store : stores) {
        storeDefinition = storeDefinitionMap.get(store);
        if (null == storeDefinition) {
            System.out.println("No store found under the name \'" + store + "\'");
            continue;
        }
        Iterator<ByteArray> keyIteratorRef = null;
        if (fetchOrphaned) {
            System.out.println("Fetching orphaned keys  of " + store);
            keyIteratorRef = adminClient.bulkFetchOps.fetchOrphanedKeys(nodeId, store);
        } else {
            System.out.println("Fetching keys in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store);
            keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(nodeId, store, partitionIdList, null, false);
        }
        File outputFile = null;
        if (directory != null) {
            outputFile = new File(directory, store + ".keys");
        }
        final Iterator<ByteArray> keyIterator = keyIteratorRef;
        if (useAscii) {
            final SerializerDefinition serializerDef = storeDefinition.getKeySerializer();
            final SerializerFactory serializerFactory = new DefaultSerializerFactory();
            @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
            final CompressionStrategy keysCompressionStrategy;
            if (serializerDef != null && serializerDef.hasCompression()) {
                keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
            } else {
                keysCompressionStrategy = null;
            }
            writeAscii(outputFile, new Writable() {

                @Override
                public void writeTo(BufferedWriter out) throws IOException {
                    while (keyIterator.hasNext()) {
                        final JsonGenerator generator = new JsonFactory(new ObjectMapper()).createJsonGenerator(out);
                        byte[] keyBytes = keyIterator.next().get();
                        Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
                        if (keyObject instanceof GenericRecord) {
                            out.write(keyObject.toString());
                        } else {
                            generator.writeObject(keyObject);
                        }
                        out.write('\n');
                    }
                }
            });
        } else {
            writeBinary(outputFile, new Printable() {

                @Override
                public void printTo(DataOutputStream out) throws IOException {
                    while (keyIterator.hasNext()) {
                        byte[] keyBytes = keyIterator.next().get();
                        out.writeChars(ByteUtils.toHexString(keyBytes) + "\n");
                    }
                }
            });
        }
        if (outputFile != null)
            System.out.println("Fetched keys from " + store + " to " + outputFile);
    }
}