Examples with DefaultSerializerFactory voldemort.serialization.DefaultSerializerFactory used on opensource projects

Example 1 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class AbstractHadoopStoreBuilderMapper method configure.

@Override
@SuppressWarnings("unchecked")
public void configure(JobConf conf) {
    super.configure(conf);
    mapper.configure(conf);
    keySerializerDefinition = getStoreDef().getKeySerializer();
    valueSerializerDefinition = getStoreDef().getValueSerializer();
    try {
        SerializerFactory factory = new DefaultSerializerFactory();
        if (conf.get("serializer.factory") != null) {
            factory = (SerializerFactory) Class.forName(conf.get("serializer.factory")).newInstance();
        }
        keySerializer = (Serializer<Object>) factory.getSerializer(keySerializerDefinition);
        valueSerializer = (Serializer<Object>) factory.getSerializer(valueSerializerDefinition);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class ReadOnlyStorageEngineTest method testIteration.

@SuppressWarnings("unchecked")
@Test
public void testIteration() throws Exception {
    ReadOnlyStorageEngineTestInstance testData = ReadOnlyStorageEngineTestInstance.create(strategy, dir, TOTAL_NUMBER_OF_RECORDS, numberOfNodes, replicationFactor, serDef, serDef, storageType, partitionMap);
    ListMultimap<Integer, Pair<String, String>> nodeToEntries = ArrayListMultimap.create();
    for (Map.Entry<String, String> entry : testData.getData().entrySet()) {
        for (Node node : testData.routeRequest(entry.getKey())) {
            nodeToEntries.put(node.getId(), Pair.create(entry.getKey(), entry.getValue()));
        }
    }
    SerializerFactory factory = new DefaultSerializerFactory();
    Serializer<String> serializer = (Serializer<String>) factory.getSerializer(serDef);
    for (Map.Entry<Integer, ReadOnlyStorageEngine> storeEntry : testData.getReadOnlyStores().entrySet()) {
        List<Pair<String, String>> entries = Lists.newArrayList(nodeToEntries.get(storeEntry.getKey()));
        ClosableIterator<ByteArray> keyIterator = null;
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = null;
        try {
            keyIterator = storeEntry.getValue().keys();
            entryIterator = storeEntry.getValue().entries();
        } catch (Exception e) {
            if (storageType.compareTo(ReadOnlyStorageFormat.READONLY_V2) == 0) {
                fail("Should not have thrown exception since this version supports iteration");
            } else {
                return;
            }
        }
        // Generate keys from entries
        List<String> keys = Lists.newArrayList();
        Iterator<Pair<String, String>> pairIterator = entries.iterator();
        while (pairIterator.hasNext()) {
            keys.add(pairIterator.next().getFirst());
        }
        // Test keys
        int keyCount = 0;
        while (keyIterator.hasNext()) {
            String key = serializer.toObject(keyIterator.next().get());
            Assert.assertEquals(keys.contains(key), true);
            keyCount++;
        }
        Assert.assertEquals(keyCount, entries.size());
        // Test entries
        int entriesCount = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Pair<String, String> stringEntry = Pair.create(serializer.toObject(entry.getFirst().get()), serializer.toObject(entry.getSecond().getValue()));
            Assert.assertEquals(entries.contains(stringEntry), true);
            entriesCount++;
        }
        Assert.assertEquals(entriesCount, entries.size());
    }
}

Example 3 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class ReadOnlyStorageEngineTestInstance method create.

public static ReadOnlyStorageEngineTestInstance create(SearchStrategy strategy, File baseDir, int testSize, int numNodes, int repFactor, SerializerDefinition keySerDef, SerializerDefinition valueSerDef, ReadOnlyStorageFormat type, int[][] partitionMap) throws Exception {
    // create some test data
    Map<String, String> data = createTestData(testSize);
    JsonReader reader = makeTestDataReader(data, baseDir);
    // set up definitions for cluster and store
    List<Node> nodes = new ArrayList<Node>();
    for (int i = 0; i < numNodes; i++) {
        List<Integer> partitions = new ArrayList<Integer>(partitionMap[i].length);
        for (int p : partitionMap[i]) {
            partitions.add(p);
        }
        nodes.add(new Node(i, "localhost", 8080 + i, 6666 + i, 7000 + i, partitions));
    }
    Cluster cluster = new Cluster("test", nodes);
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(keySerDef).setValueSerializer(valueSerDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(repFactor).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    // build store files in outputDir
    File outputDir = TestUtils.createTempDir(baseDir);
    JsonStoreBuilder storeBuilder = new JsonStoreBuilder(reader, cluster, storeDef, router, outputDir, null, testSize / 5, 1, 2, 10000, false);
    storeBuilder.build(type);
    File nodeDir = TestUtils.createTempDir(baseDir);
    @SuppressWarnings("unchecked") Serializer<String> keySerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(keySerDef);
    @SuppressWarnings("unchecked") Serializer<String> valueSerializer = (Serializer<String>) new DefaultSerializerFactory().getSerializer(valueSerDef);
    Serializer<String> transSerializer = new StringSerializer();
    Map<Integer, Store<String, String, String>> nodeStores = Maps.newHashMap();
    Map<Integer, ReadOnlyStorageEngine> readOnlyStores = Maps.newHashMap();
    for (int i = 0; i < numNodes; i++) {
        File currNode = new File(nodeDir, Integer.toString(i));
        currNode.mkdirs();
        currNode.deleteOnExit();
        Utils.move(new File(outputDir, "node-" + Integer.toString(i)), new File(currNode, "version-0"));
        CompressionStrategyFactory compressionStrategyFactory = new CompressionStrategyFactory();
        CompressionStrategy keyCompressionStrat = compressionStrategyFactory.get(keySerDef.getCompression());
        CompressionStrategy valueCompressionStrat = compressionStrategyFactory.get(valueSerDef.getCompression());
        ReadOnlyStorageEngine readOnlyStorageEngine = new ReadOnlyStorageEngine("test", strategy, router, i, currNode, 1);
        readOnlyStores.put(i, readOnlyStorageEngine);
        Store<ByteArray, byte[], byte[]> innerStore = new CompressingStore(readOnlyStorageEngine, keyCompressionStrat, valueCompressionStrat);
        nodeStores.put(i, SerializingStore.wrap(innerStore, keySerializer, valueSerializer, transSerializer));
    }
    return new ReadOnlyStorageEngineTestInstance(data, baseDir, readOnlyStores, nodeStores, router, keySerializer);
}

Example 4 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class HadoopStoreBuilderCollisionTest method testCollisionWithParams.

@SuppressWarnings({ "unchecked" })
public void testCollisionWithParams(int totalElements, int maxCollisions) throws Exception {
    assertEquals(totalElements % maxCollisions, 0);
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    List<String> valuesLeft = Lists.newArrayList();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp");
    File outputDir = new File(testDir, "output");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < totalElements; i++) {
        values.put(Integer.toString(i), Integer.toString(i));
        valuesLeft.add(Integer.toString(i));
    }
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    byte[] currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
    int entryId = 0;
    for (Map.Entry<String, String> entry : values.entrySet()) {
        if (entryId % maxCollisions == 0) {
            currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
        }
        contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
        byte[] oldMd5 = ByteUtils.copy(ByteUtils.md5(serializer.toBytes(entry.getKey())), 0, 2 * ByteUtils.SIZE_OF_INT);
        oldMd5ToNewMd5.put(new ByteArray(oldMd5), currentMd5);
        entryId++;
    }
    FileUtils.writeStringToFile(inputFile, contents.toString());
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testCollisionWithParams", new Props(), new JobConf(), CollidingTextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, true, false, 1024 * 1024 * 1024, false, null, false);
    builder.build();
    File nodeFile = new File(outputDir, "node-0");
    File versionDir = new File(storeDir, "version-0");
    HdfsFetcher fetcher = new HdfsFetcher();
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    // Test if we work in the normal collision scenario open store
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, new CustomBinarySearchStrategy(), new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        List<Versioned<Object>> found = store.get(entry.getKey(), null);
        Assert.assertEquals("Incorrect number of results", 1, found.size());
        Assert.assertEquals(entry.getValue(), found.get(0).getValue());
    }
    // also check the iterator - first key iterator...
    List<String> valuesLeft2 = Lists.newArrayList(valuesLeft);
    ClosableIterator<ByteArray> keyIterator = engine.keys();
    int numElements = 0;
    while (keyIterator.hasNext()) {
        Object object = serializer.toObject(keyIterator.next().get());
        assertEquals(valuesLeft.remove(object), true);
        Assert.assertTrue(values.containsKey(object));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft.size(), 0);
    // ... and entry iterator
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
    numElements = 0;
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        assertEquals(valuesLeft2.remove(serializer.toObject(entry.getFirst().get())), true);
        Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft2.size(), 0);
}

Example 5 with DefaultSerializerFactory

use of voldemort.serialization.DefaultSerializerFactory in project voldemort by voldemort.

the class RequestFileFilter method filter.

// TODO: support keys other than integer or string, general cleanup
public void filter() throws IOException {
    SerializerFactory factory = new DefaultSerializerFactory();
    @SuppressWarnings("unchecked") Serializer<Object> keySerializer = (Serializer<Object>) factory.getSerializer(storeDefinition.getKeySerializer());
    BufferedReader in = new BufferedReader(new FileReader(inputFile));
    BufferedWriter out = new BufferedWriter(new FileWriter(outputFile));
    try {
        String line = null;
        while ((line = in.readLine()) != null) {
            String keyStr = line.replaceAll("\\s+$", "");
            Object key = null;
            if (stringKeys) {
                key = keyStr;
            } else {
                key = Integer.valueOf(keyStr);
            }
            byte[] keyBytes = keySerializer.toBytes(key);
            List<Node> nodes = routingStrategy.routeRequest(keyBytes);
            if (nodes.contains(node)) {
                out.write(key + "\n");
            }
        }
    } finally {
        try {
            in.close();
        } finally {
            out.close();
        }
    }
}