Examples with CompressionStrategy voldemort.store.compress.CompressionStrategy used on opensource projects

Example 1 with CompressionStrategy

use of voldemort.store.compress.CompressionStrategy in project voldemort by voldemort.

the class ReadOnlyStorageEngineTestInstance method create.

public static ReadOnlyStorageEngineTestInstance create(SearchStrategy strategy, File baseDir, int testSize, int numNodes, int repFactor, SerializerDefinition keySerDef, SerializerDefinition valueSerDef, ReadOnlyStorageFormat type, int[][] partitionMap) throws Exception {
    // create some test data
    Map<String, String> data = createTestData(testSize);
    JsonReader reader = makeTestDataReader(data, baseDir);
    // set up definitions for cluster and store
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < numNodes; i++) {
        List<Integer> partitions = new ArrayList<Integer>(partitionMap[i].length);
        for (int p : partitionMap[i]) {
            partitions.add(p);
        }
        nodes.add(new Node(i, "localhost", 8080 + i, 6666 + i, 7000 + i, partitions));
    }
    Cluster cluster = new Cluster("test", nodes);
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(keySerDef).setValueSerializer(valueSerDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(repFactor).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // build store files in outputDir
    File outputDir = TestUtils.createTempDir(baseDir);
    JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testSize / 5, 1, 2, 10000, false);
    storeBuilder.build(type);
    File nodeDir = TestUtils.createTempDir(baseDir);
    @SuppressWarnings("unchecked") Serializer<String> keySerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(keySerDef);
    @SuppressWarnings("unchecked") Serializer<String> valueSerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(valueSerDef);
    Serializer<String> transSerializer = new StringSerializer();
    Map<Integer, Store<String, String, String>> nodeStores = Maps.newHashMap();
    Map<Integer, ReadOnlyStorageEngine> readOnlyStores = Maps.newHashMap();
    for (int i = 0; i < numNodes; i++) {
        File currNode = new File(nodeDir, Integer.toString(i));
        currNode.mkdirs();
        currNode.deleteOnExit();
        Utils.move(new File(outputDir, "node-" + Integer.toString(i)), new File(currNode, "version-0"));
        CompressionStrategyFactory compressionStrategyFactory = new CompressionStrategyFactory();
        CompressionStrategy keyCompressionStrat = compressionStrategyFactory.get(keySerDef.getCompression());
        CompressionStrategy valueCompressionStrat = compressionStrategyFactory.get(valueSerDef.getCompression());
        ReadOnlyStorageEngine readOnlyStorageEngine = new ReadOnlyStorageEngine("test", strategy, router, i, currNode, 1);
        readOnlyStores.put(i, readOnlyStorageEngine);
        Store<ByteArray, byte[], byte[]> innerStore = new CompressingStore(readOnlyStorageEngine, keyCompressionStrat, valueCompressionStrat);
        nodeStores.put(i, SerializingStore.wrap(innerStore, keySerializer, valueSerializer, transSerializer));
    }
    return new ReadOnlyStorageEngineTestInstance(data, baseDir, readOnlyStores, nodeStores, router, keySerializer);
}

Example 2 with CompressionStrategy

use of voldemort.store.compress.CompressionStrategy in project voldemort by voldemort.

the class ViewStorageConfiguration method getStore.

public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    String name = storeDef.getName();
    StoreDefinition def = StoreUtils.getStoreDef(storeDefs, name);
    String targetName = def.getViewTargetStoreName();
    StoreDefinition targetDef = StoreUtils.getStoreDef(storeDefs, targetName);
    StorageEngine<ByteArray, byte[], byte[]> target = storeRepo.getStorageEngine(targetName);
    if (target == null)
        throw new VoldemortException("View \"" + name + "\" has a target store \"" + targetName + "\" which does not exist.");
    String factoryName = def.getSerializerFactory();
    SerializerFactory factory;
    if (factoryName == null)
        factory = new DefaultSerializerFactory();
    else
        factory = loadSerializerFactory(factoryName);
    CompressionStrategy valueCompressionStrategy = null;
    if (targetDef.getValueSerializer().hasCompression()) {
        valueCompressionStrategy = new CompressionStrategyFactory().get(targetDef.getValueSerializer().getCompression());
    }
    View<?, ?, ?, ?> view = loadTransformation(def.getValueTransformation());
    return new ViewStorageEngine(name, target, factory.getSerializer(def.getValueSerializer()), def.getTransformsSerializer() != null ? factory.getSerializer(def.getTransformsSerializer()) : null, factory.getSerializer(targetDef.getKeySerializer()), factory.getSerializer(targetDef.getValueSerializer()), valueCompressionStrategy, view);
}

Example 3 with CompressionStrategy

use of voldemort.store.compress.CompressionStrategy in project voldemort by voldemort.

the class StreamingClientTest method verifyKeysExist.

/*
     * Checks if each node has the keys it is reponsible for returns false
     * otherwise
     */
public boolean verifyKeysExist(int nodeIdToVerifyOn) {
    RoutingStrategyFactory factory = new RoutingStrategyFactory();
    RoutingStrategy storeRoutingStrategy = factory.updateRoutingStrategy(storeDef, adminClient.getAdminClientCluster());
    HashMap<Integer, ArrayList<String>> expectedNodeIdToKeys;
    expectedNodeIdToKeys = new HashMap();
    Collection<Node> nodesInCluster = adminClient.getAdminClientCluster().getNodes();
    for (Node node : nodesInCluster) {
        ArrayList<String> keysForNode = new ArrayList();
        expectedNodeIdToKeys.put(node.getId(), keysForNode);
    }
    for (int i = 0; i < NUM_KEYS_1; i++) {
        String key = i + "";
        String value = key;
        List<Node> nodeList = storeRoutingStrategy.routeRequest(key.getBytes());
        for (Node node : nodeList) {
            ArrayList<String> keysForNode = expectedNodeIdToKeys.get(node.getId());
            keysForNode.add(key);
        }
    }
    ArrayList<String> fetchedKeysForNode = new ArrayList();
    for (Node node : nodesInCluster) {
        List<Integer> partitionIdList = Lists.newArrayList();
        partitionIdList.addAll(node.getPartitionIds());
        Iterator<ByteArray> keyIteratorRef = null;
        keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(node.getId(), TEST_STORE_NAME, partitionIdList, null, false);
        final SerializerDefinition serializerDef = storeDef.getKeySerializer();
        final SerializerFactory serializerFactory = new DefaultSerializerFactory();
        @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
        final CompressionStrategy keysCompressionStrategy;
        if (serializerDef != null && serializerDef.hasCompression()) {
            keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
        } else {
            keysCompressionStrategy = null;
        }
        final Iterator<ByteArray> keyIterator = keyIteratorRef;
        while (keyIterator.hasNext()) {
            byte[] keyBytes = keyIterator.next().get();
            try {
                Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
                fetchedKeysForNode.add((String) keyObject);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    ArrayList<String> keysForNode = expectedNodeIdToKeys.get(nodeIdToVerifyOn);
    if (!fetchedKeysForNode.containsAll(keysForNode))
        return false;
    else
        return true;
}

Example 4 with CompressionStrategy

use of voldemort.store.compress.CompressionStrategy in project voldemort by voldemort.

the class VoldemortAdminTool method executeFetchKeys.

private static void executeFetchKeys(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException {
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    Map<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap();
    for (StoreDefinition storeDefinition : storeDefinitionList) {
        storeDefinitionMap.put(storeDefinition.getName(), storeDefinition);
    }
    File directory = null;
    if (outputDir != null) {
        directory = new File(outputDir);
        if (!(directory.exists() || directory.mkdir())) {
            Utils.croak("Can't find or create directory " + outputDir);
        }
    }
    List<String> stores = storeNames;
    if (stores == null) {
        stores = Lists.newArrayList();
        stores.addAll(storeDefinitionMap.keySet());
    } else {
        // add system stores to the map so they can be fetched when
        // specified explicitly
        storeDefinitionMap.putAll(getSystemStoreDefs());
    }
    // Pick up all the partitions
    if (partitionIdList == null) {
        partitionIdList = Lists.newArrayList();
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            partitionIdList.addAll(node.getPartitionIds());
        }
    }
    StoreDefinition storeDefinition = null;
    for (String store : stores) {
        storeDefinition = storeDefinitionMap.get(store);
        if (null == storeDefinition) {
            System.out.println("No store found under the name \'" + store + "\'");
            continue;
        }
        Iterator<ByteArray> keyIteratorRef = null;
        if (fetchOrphaned) {
            System.out.println("Fetching orphaned keys  of " + store);
            keyIteratorRef = adminClient.bulkFetchOps.fetchOrphanedKeys(nodeId, store);
        } else {
            System.out.println("Fetching keys in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store);
            keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(nodeId, store, partitionIdList, null, false);
        }
        File outputFile = null;
        if (directory != null) {
            outputFile = new File(directory, store + ".keys");
        }
        final Iterator<ByteArray> keyIterator = keyIteratorRef;
        if (useAscii) {
            final SerializerDefinition serializerDef = storeDefinition.getKeySerializer();
            final SerializerFactory serializerFactory = new DefaultSerializerFactory();
            @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef);
            final CompressionStrategy keysCompressionStrategy;
            if (serializerDef != null && serializerDef.hasCompression()) {
                keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression());
            } else {
                keysCompressionStrategy = null;
            }
            writeAscii(outputFile, new Writable() {

                @Override
                public void writeTo(BufferedWriter out) throws IOException {
                    while (keyIterator.hasNext()) {
                        final JsonGenerator generator = new JsonFactory(new ObjectMapper()).createJsonGenerator(out);
                        byte[] keyBytes = keyIterator.next().get();
                        Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes));
                        if (keyObject instanceof GenericRecord) {
                            out.write(keyObject.toString());
                        } else {
                            generator.writeObject(keyObject);
                        }
                        out.write('\n');
                    }
                }
            });
        } else {
            writeBinary(outputFile, new Printable() {

                @Override
                public void printTo(DataOutputStream out) throws IOException {
                    while (keyIterator.hasNext()) {
                        byte[] keyBytes = keyIterator.next().get();
                        out.writeChars(ByteUtils.toHexString(keyBytes) + "\n");
                    }
                }
            });
        }
        if (outputFile != null)
            System.out.println("Fetched keys from " + store + " to " + outputFile);
    }
}

Example 5 with CompressionStrategy

use of voldemort.store.compress.CompressionStrategy in project voldemort by voldemort.

the class VoldemortAdminTool method executeFetchEntries.

private static void executeFetchEntries(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException {
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    HashMap<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap();
    for (StoreDefinition storeDefinition : storeDefinitionList) {
        storeDefinitionMap.put(storeDefinition.getName(), storeDefinition);
    }
    File directory = null;
    if (outputDir != null) {
        directory = new File(outputDir);
        if (!(directory.exists() || directory.mkdir())) {
            Utils.croak("Can't find or create directory " + outputDir);
        }
    }
    List<String> stores = storeNames;
    if (stores == null) {
        // when no stores specified, all user defined store will be fetched,
        // but not system stores.
        stores = Lists.newArrayList();
        stores.addAll(storeDefinitionMap.keySet());
    } else {
        // add system stores to the map so they can be fetched when
        // specified explicitly
        storeDefinitionMap.putAll(getSystemStoreDefs());
    }
    // Pick up all the partitions
    if (partitionIdList == null) {
        partitionIdList = Lists.newArrayList();
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            partitionIdList.addAll(node.getPartitionIds());
        }
    }
    StoreDefinition storeDefinition = null;
    for (String store : stores) {
        storeDefinition = storeDefinitionMap.get(store);
        if (null == storeDefinition) {
            System.out.println("No store found under the name \'" + store + "\'");
            continue;
        }
        Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIteratorRef = null;
        if (fetchOrphaned) {
            System.out.println("Fetching orphaned entries of " + store);
            entriesIteratorRef = adminClient.bulkFetchOps.fetchOrphanedEntries(nodeId, store);
        } else {
            System.out.println("Fetching entries in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store);
            entriesIteratorRef = adminClient.bulkFetchOps.fetchEntries(nodeId, store, partitionIdList, null, false);
        }
        final Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = entriesIteratorRef;
        File outputFile = null;
        if (directory != null) {
            outputFile = new File(directory, store + ".entries");
        }
        if (useAscii) {
            // k-v serializer
            SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
            SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
            SerializerFactory serializerFactory = new DefaultSerializerFactory();
            @SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
            @SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);
            // compression strategy
            final CompressionStrategy keyCompressionStrategy;
            final CompressionStrategy valueCompressionStrategy;
            if (keySerializerDef != null && keySerializerDef.hasCompression()) {
                keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
            } else {
                keyCompressionStrategy = null;
            }
            if (valueSerializerDef != null && valueSerializerDef.hasCompression()) {
                valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
            } else {
                valueCompressionStrategy = null;
            }
            writeAscii(outputFile, new Writable() {

                @Override
                public void writeTo(BufferedWriter out) throws IOException {
                    while (entriesIterator.hasNext()) {
                        final JsonGenerator generator = new JsonFactory(new ObjectMapper()).createJsonGenerator(out);
                        Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next();
                        byte[] keyBytes = kvPair.getFirst().get();
                        byte[] valueBytes = kvPair.getSecond().getValue();
                        VectorClock version = (VectorClock) kvPair.getSecond().getVersion();
                        Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes));
                        Object valueObject = valueSerializer.toObject((null == valueCompressionStrategy) ? valueBytes : valueCompressionStrategy.inflate(valueBytes));
                        if (keyObject instanceof GenericRecord) {
                            out.write(keyObject.toString());
                        } else {
                            generator.writeObject(keyObject);
                        }
                        out.write(' ' + version.toString() + ' ');
                        if (valueObject instanceof GenericRecord) {
                            out.write(valueObject.toString());
                        } else {
                            generator.writeObject(valueObject);
                        }
                        out.write('\n');
                    }
                }
            });
        } else {
            writeBinary(outputFile, new Printable() {

                @Override
                public void printTo(DataOutputStream out) throws IOException {
                    while (entriesIterator.hasNext()) {
                        Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next();
                        byte[] keyBytes = kvPair.getFirst().get();
                        VectorClock clock = ((VectorClock) kvPair.getSecond().getVersion());
                        byte[] valueBytes = kvPair.getSecond().getValue();
                        out.writeChars(ByteUtils.toHexString(keyBytes));
                        out.writeChars(",");
                        out.writeChars(clock.toString());
                        out.writeChars(",");
                        out.writeChars(ByteUtils.toHexString(valueBytes));
                        out.writeChars("\n");
                    }
                }
            });
        }
        if (outputFile != null)
            System.out.println("Fetched keys from " + store + " to " + outputFile);
    }
}