Examples with Serializer voldemort.serialization.Serializer used on opensource projects

Example 1 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class RESTClientFactory method getRawStore.

@Override
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver) {
    Store<K, V, T> clientStore = null;
    // The lowest layer : Transporting request to coordinator
    R2Store r2store = null;
    this.d2Client = restClientFactoryConfig.getD2Client();
    if (this.d2Client == null) {
        logger.info("Using transportclient since d2client is not available");
        Map<String, String> properties = new HashMap<String, String>();
        properties.put(HttpClientFactory.HTTP_POOL_SIZE, Integer.toString(this.config.getMaxR2ConnectionPoolSize()));
        transportClient = _clientFactory.getClient(properties);
        r2store = new R2Store(storeName, this.config.getHttpBootstrapURL(), this.transportClient, this.config);
    } else {
        logger.info("Using d2client");
        r2store = new R2Store(storeName, this.config.getHttpBootstrapURL(), this.d2Client, this.config);
    }
    this.rawStoreList.add(r2store);
    // bootstrap from the coordinator and obtain all the serialization
    // information.
    String serializerInfoXml = r2store.getSerializerInfoXml();
    SerializerDefinition keySerializerDefinition = RestUtils.parseKeySerializerDefinition(serializerInfoXml);
    SerializerDefinition valueSerializerDefinition = RestUtils.parseValueSerializerDefinition(serializerInfoXml);
    synchronized (this) {
        keySerializerMap.put(storeName, keySerializerDefinition);
        valueSerializerMap.put(storeName, valueSerializerDefinition);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Bootstrapping for " + storeName + ": Key serializer " + keySerializerDefinition);
        logger.debug("Bootstrapping for " + storeName + ": Value serializer " + valueSerializerDefinition);
    }
    // Start building the stack..
    // First, the transport layer
    Store<ByteArray, byte[], byte[]> store = r2store;
    // TODO: Add identifierString to the Mbean name
    if (this.config.isEnableJmx()) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.stats);
        store = statStore;
        JmxUtils.registerMbean(new StoreStatsJmx(statStore.getStats()), JmxUtils.createObjectName(JmxUtils.getPackageName(store.getClass()), store.getName()));
    }
    // Add compression layer
    if (keySerializerDefinition.hasCompression() || valueSerializerDefinition.hasCompression()) {
        store = new CompressingStore(store, new CompressionStrategyFactory().get(keySerializerDefinition.getCompression()), new CompressionStrategyFactory().get(valueSerializerDefinition.getCompression()));
    }
    // Add Serialization layer
    Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(keySerializerDefinition);
    Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(valueSerializerDefinition);
    clientStore = SerializingStore.wrap(store, keySerializer, valueSerializer, null);
    // Add inconsistency Resolving layer
    InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver<V>() : resolver;
    clientStore = new InconsistencyResolvingStore<K, V, T>(clientStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver<V>(), secondaryResolver));
    return clientStore;
}

Example 2 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class HadoopStoreBuilderCollisionTest method testCollisionWithParams.

@SuppressWarnings({ "unchecked" })
public void testCollisionWithParams(int totalElements, int maxCollisions) throws Exception {
    assertEquals(totalElements % maxCollisions, 0);
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    List<String> valuesLeft = Lists.newArrayList();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp");
    File outputDir = new File(testDir, "output");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < totalElements; i++) {
        values.put(Integer.toString(i), Integer.toString(i));
        valuesLeft.add(Integer.toString(i));
    }
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    byte[] currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
    int entryId = 0;
    for (Map.Entry<String, String> entry : values.entrySet()) {
        if (entryId % maxCollisions == 0) {
            currentMd5 = TestUtils.randomBytes(2 * ByteUtils.SIZE_OF_INT);
        }
        contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
        byte[] oldMd5 = ByteUtils.copy(ByteUtils.md5(serializer.toBytes(entry.getKey())), 0, 2 * ByteUtils.SIZE_OF_INT);
        oldMd5ToNewMd5.put(new ByteArray(oldMd5), currentMd5);
        entryId++;
    }
    FileUtils.writeStringToFile(inputFile, contents.toString());
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testCollisionWithParams", new Props(), new JobConf(), CollidingTextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, true, false, 1024 * 1024 * 1024, false, null, false);
    builder.build();
    File nodeFile = new File(outputDir, "node-0");
    File versionDir = new File(storeDir, "version-0");
    HdfsFetcher fetcher = new HdfsFetcher();
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    // Test if we work in the normal collision scenario open store
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, new CustomBinarySearchStrategy(), new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        List<Versioned<Object>> found = store.get(entry.getKey(), null);
        Assert.assertEquals("Incorrect number of results", 1, found.size());
        Assert.assertEquals(entry.getValue(), found.get(0).getValue());
    }
    // also check the iterator - first key iterator...
    List<String> valuesLeft2 = Lists.newArrayList(valuesLeft);
    ClosableIterator<ByteArray> keyIterator = engine.keys();
    int numElements = 0;
    while (keyIterator.hasNext()) {
        Object object = serializer.toObject(keyIterator.next().get());
        assertEquals(valuesLeft.remove(object), true);
        Assert.assertTrue(values.containsKey(object));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft.size(), 0);
    // ... and entry iterator
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
    numElements = 0;
    while (entryIterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
        assertEquals(valuesLeft2.remove(serializer.toObject(entry.getFirst().get())), true);
        Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
        numElements++;
    }
    Assert.assertEquals(numElements, values.size());
    Assert.assertEquals(valuesLeft2.size(), 0);
}

Example 3 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class RequestFileFilter method filter.

// TODO: support keys other than integer or string, general cleanup
public void filter() throws IOException {
    SerializerFactory factory = new DefaultSerializerFactory();
    @SuppressWarnings("unchecked") Serializer<Object> keySerializer = (Serializer<Object>) factory.getSerializer(storeDefinition.getKeySerializer());
    BufferedReader in = new BufferedReader(new FileReader(inputFile));
    BufferedWriter out = new BufferedWriter(new FileWriter(outputFile));
    try {
        String line = null;
        while ((line = in.readLine()) != null) {
            String keyStr = line.replaceAll("\\s+$", "");
            Object key = null;
            if (stringKeys) {
                key = keyStr;
            } else {
                key = Integer.valueOf(keyStr);
            }
            byte[] keyBytes = keySerializer.toBytes(key);
            List<Node> nodes = routingStrategy.routeRequest(keyBytes);
            if (nodes.contains(node)) {
                out.write(key + "\n");
            }
        }
    } finally {
        try {
            in.close();
        } finally {
            out.close();
        }
    }
}

Example 4 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class Benchmark method initializeStore.

@SuppressWarnings("unchecked")
public void initializeStore(Props benchmarkProps) throws Exception {
    this.numThreads = benchmarkProps.getInt(THREADS, MAX_WORKERS);
    this.numConnectionsPerNode = benchmarkProps.getInt(NUM_CONNECTIONS_PER_NODE, MAX_CONNECTIONS_PER_NODE);
    this.numIterations = benchmarkProps.getInt(ITERATIONS, 1);
    this.statusIntervalSec = benchmarkProps.getInt(INTERVAL, 0);
    this.verbose = benchmarkProps.getBoolean(VERBOSE, false);
    this.verifyRead = benchmarkProps.getBoolean(VERIFY, false);
    this.ignoreNulls = benchmarkProps.getBoolean(IGNORE_NULLS, false);
    int clientZoneId = benchmarkProps.getInt(CLIENT_ZONE_ID, -1);
    if (benchmarkProps.containsKey(URL)) {
        // Remote benchmark
        if (!benchmarkProps.containsKey(STORE_NAME)) {
            throw new VoldemortException("Missing storename");
        }
        String socketUrl = benchmarkProps.getString(URL);
        String storeName = benchmarkProps.getString(STORE_NAME);
        ClientConfig clientConfig = new ClientConfig().setMaxThreads(numThreads).setMaxTotalConnections(numThreads).setMaxConnectionsPerNode(numConnectionsPerNode).setRoutingTimeout(1500, TimeUnit.MILLISECONDS).setSocketTimeout(1500, TimeUnit.MILLISECONDS).setConnectionTimeout(500, TimeUnit.MILLISECONDS).setRequestFormatType(RequestFormatType.VOLDEMORT_V3).setBootstrapUrls(socketUrl);
        if (clientZoneId >= 0) {
            clientConfig.setClientZoneId(clientZoneId);
        }
        SocketStoreClientFactory socketFactory = new SocketStoreClientFactory(clientConfig);
        this.storeClient = socketFactory.getStoreClient(storeName);
        StoreDefinition storeDef = getStoreDefinition(socketFactory, storeName);
        this.keyType = findKeyType(storeDef);
        benchmarkProps.put(Benchmark.KEY_TYPE, this.keyType);
        this.factory = socketFactory;
    } else {
        // Local benchmark
        localMode = true;
        String storageEngineClass = benchmarkProps.getString(STORAGE_CONFIGURATION_CLASS);
        this.keyType = benchmarkProps.getString(KEY_TYPE, STRING_KEY_TYPE);
        Serializer serializer = findKeyType(this.keyType);
        Store<Object, Object, Object> store = null;
        VoldemortConfig voldemortConfig;
        if (benchmarkProps.containsKey(LOCAL_SERVER_PROPERTIES)) {
            File homeDir = TestUtils.createTempDir();
            File configDir = new File(homeDir, "config");
            configDir.mkdir();
            FileUtils.copyFile(new File(benchmarkProps.get(LOCAL_SERVER_PROPERTIES)), new File(configDir, "server.properties"));
            voldemortConfig = VoldemortConfig.loadFromVoldemortHome(homeDir.getAbsolutePath());
        } else {
            voldemortConfig = ServerTestUtils.getVoldemortConfig();
        }
        StorageConfiguration conf = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { voldemortConfig });
        StorageEngine<ByteArray, byte[], byte[]> engine = conf.getStore(TestUtils.makeStoreDefinition(DUMMY_DB), TestUtils.makeSingleNodeRoutingStrategy());
        if (conf.getType().compareTo(ViewStorageConfiguration.TYPE_NAME) == 0) {
            engine = new ViewStorageEngine(STORE_NAME, engine, new StringSerializer(), new StringSerializer(), serializer, new StringSerializer(), null, BenchmarkViews.loadTransformation(benchmarkProps.getString(VIEW_CLASS).trim()));
        }
        store = SerializingStore.wrap(engine, serializer, new StringSerializer(), new IdentitySerializer());
        this.factory = new StaticStoreClientFactory(store);
        this.storeClient = factory.getStoreClient(store.getName());
    }
    this.storeInitialized = true;
}

Example 5 with Serializer

use of voldemort.serialization.Serializer in project voldemort by voldemort.

the class ReadOnlyStorageEngineTest method testIteration.

@SuppressWarnings("unchecked")
@Test
public void testIteration() throws Exception {
    ReadOnlyStorageEngineTestInstance testData = ReadOnlyStorageEngineTestInstance.create(strategy, dir, TOTAL_NUMBER_OF_RECORDS, numberOfNodes, replicationFactor, serDef, serDef, storageType, partitionMap);
    ListMultimap<Integer, Pair<String, String>> nodeToEntries = ArrayListMultimap.create();
    for (Map.Entry<String, String> entry : testData.getData().entrySet()) {
        for (Node node : testData.routeRequest(entry.getKey())) {
            nodeToEntries.put(node.getId(), Pair.create(entry.getKey(), entry.getValue()));
        }
    }
    SerializerFactory factory = new DefaultSerializerFactory();
    Serializer<String> serializer = (Serializer<String>) factory.getSerializer(serDef);
    for (Map.Entry<Integer, ReadOnlyStorageEngine> storeEntry : testData.getReadOnlyStores().entrySet()) {
        List<Pair<String, String>> entries = Lists.newArrayList(nodeToEntries.get(storeEntry.getKey()));
        ClosableIterator<ByteArray> keyIterator = null;
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = null;
        try {
            keyIterator = storeEntry.getValue().keys();
            entryIterator = storeEntry.getValue().entries();
        } catch (Exception e) {
            if (storageType.compareTo(ReadOnlyStorageFormat.READONLY_V2) == 0) {
                fail("Should not have thrown exception since this version supports iteration");
            } else {
                return;
            }
        }
        // Generate keys from entries
        List<String> keys = Lists.newArrayList();
        Iterator<Pair<String, String>> pairIterator = entries.iterator();
        while (pairIterator.hasNext()) {
            keys.add(pairIterator.next().getFirst());
        }
        // Test keys
        int keyCount = 0;
        while (keyIterator.hasNext()) {
            String key = serializer.toObject(keyIterator.next().get());
            Assert.assertEquals(keys.contains(key), true);
            keyCount++;
        }
        Assert.assertEquals(keyCount, entries.size());
        // Test entries
        int entriesCount = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Pair<String, String> stringEntry = Pair.create(serializer.toObject(entry.getFirst().get()), serializer.toObject(entry.getSecond().getValue()));
            Assert.assertEquals(entries.contains(stringEntry), true);
            entriesCount++;
        }
        Assert.assertEquals(entriesCount, entries.size());
    }
}