Examples with SerializerFactory voldemort.serialization.SerializerFactory used on opensource projects

Example 1 with SerializerFactory

use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.

the class AbstractHadoopStoreBuilderMapper method configure.

@Override
@SuppressWarnings("unchecked")
public void configure(JobConf conf) {
    super.configure(conf);
    mapper.configure(conf);
    keySerializerDefinition = getStoreDef().getKeySerializer();
    valueSerializerDefinition = getStoreDef().getValueSerializer();
    try {
        SerializerFactory factory = new DefaultSerializerFactory();
        if (conf.get("serializer.factory") != null) {
            factory = (SerializerFactory) Class.forName(conf.get("serializer.factory")).newInstance();
        }
        keySerializer = (Serializer<Object>) factory.getSerializer(keySerializerDefinition);
        valueSerializer = (Serializer<Object>) factory.getSerializer(valueSerializerDefinition);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with SerializerFactory

use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.

the class ReadOnlyStorageEngineTest method testIteration.

@SuppressWarnings("unchecked")
@Test
public void testIteration() throws Exception {
    ReadOnlyStorageEngineTestInstance testData = ReadOnlyStorageEngineTestInstance.create(strategy, dir, TOTAL_NUMBER_OF_RECORDS, numberOfNodes, replicationFactor, serDef, serDef, storageType, partitionMap);
    ListMultimap<Integer, Pair<String, String>> nodeToEntries = ArrayListMultimap.create();
    for (Map.Entry<String, String> entry : testData.getData().entrySet()) {
        for (Node node : testData.routeRequest(entry.getKey())) {
            nodeToEntries.put(node.getId(), Pair.create(entry.getKey(), entry.getValue()));
        }
    }
    SerializerFactory factory = new DefaultSerializerFactory();
    Serializer<String> serializer = (Serializer<String>) factory.getSerializer(serDef);
    for (Map.Entry<Integer, ReadOnlyStorageEngine> storeEntry : testData.getReadOnlyStores().entrySet()) {
        List<Pair<String, String>> entries = Lists.newArrayList(nodeToEntries.get(storeEntry.getKey()));
        ClosableIterator<ByteArray> keyIterator = null;
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = null;
        try {
            keyIterator = storeEntry.getValue().keys();
            entryIterator = storeEntry.getValue().entries();
        } catch (Exception e) {
            if (storageType.compareTo(ReadOnlyStorageFormat.READONLY_V2) == 0) {
                fail("Should not have thrown exception since this version supports iteration");
            } else {
                return;
            }
        }
        // Generate keys from entries
        List<String> keys = Lists.newArrayList();
        Iterator<Pair<String, String>> pairIterator = entries.iterator();
        while (pairIterator.hasNext()) {
            keys.add(pairIterator.next().getFirst());
        }
        // Test keys
        int keyCount = 0;
        while (keyIterator.hasNext()) {
            String key = serializer.toObject(keyIterator.next().get());
            Assert.assertEquals(keys.contains(key), true);
            keyCount++;
        }
        Assert.assertEquals(keyCount, entries.size());
        // Test entries
        int entriesCount = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Pair<String, String> stringEntry = Pair.create(serializer.toObject(entry.getFirst().get()), serializer.toObject(entry.getSecond().getValue()));
            Assert.assertEquals(entries.contains(stringEntry), true);
            entriesCount++;
        }
        Assert.assertEquals(entriesCount, entries.size());
    }
}

Example 3 with SerializerFactory

use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.

the class RequestFileFilter method filter.

// TODO: support keys other than integer or string, general cleanup
public void filter() throws IOException {
    SerializerFactory factory = new DefaultSerializerFactory();
    @SuppressWarnings("unchecked") Serializer<Object> keySerializer = (Serializer<Object>) factory.getSerializer(storeDefinition.getKeySerializer());
    BufferedReader in = new BufferedReader(new FileReader(inputFile));
    BufferedWriter out = new BufferedWriter(new FileWriter(outputFile));
    try {
        String line = null;
        while ((line = in.readLine()) != null) {
            String keyStr = line.replaceAll("\\s+$", "");
            Object key = null;
            if (stringKeys) {
                key = keyStr;
            } else {
                key = Integer.valueOf(keyStr);
            }
            byte[] keyBytes = keySerializer.toBytes(key);
            List<Node> nodes = routingStrategy.routeRequest(keyBytes);
            if (nodes.contains(node)) {
                out.write(key + "\n");
            }
        }
    } finally {
        try {
            in.close();
        } finally {
            out.close();
        }
    }
}

Example 4 with SerializerFactory

use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.

the class ClientConfig method setProperties.

private void setProperties(Properties properties) {
    Props props = new Props(properties);
    if (props.containsKey(MAX_CONNECTIONS_PER_NODE_PROPERTY))
        this.setMaxConnectionsPerNode(props.getInt(MAX_CONNECTIONS_PER_NODE_PROPERTY));
    if (props.containsKey(MAX_TOTAL_CONNECTIONS_PROPERTY))
        this.setMaxTotalConnections(props.getInt(MAX_TOTAL_CONNECTIONS_PROPERTY));
    if (props.containsKey(MAX_THREADS_PROPERTY))
        this.setMaxThreads(props.getInt(MAX_THREADS_PROPERTY));
    if (props.containsKey(MAX_QUEUED_REQUESTS_PROPERTY))
        this.setMaxQueuedRequests(props.getInt(MAX_QUEUED_REQUESTS_PROPERTY));
    if (props.containsKey(THREAD_IDLE_MS_PROPERTY))
        this.setThreadIdleTime(props.getLong(THREAD_IDLE_MS_PROPERTY), TimeUnit.MILLISECONDS);
    if (props.containsKey(CONNECTION_TIMEOUT_MS_PROPERTY))
        this.setConnectionTimeout(props.getInt(CONNECTION_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
    if (props.containsKey(IDLE_CONNECTION_TIMEOUT_MINUTES_PROPERTY))
        this.setIdleConnectionTimeout(props.getInt(IDLE_CONNECTION_TIMEOUT_MINUTES_PROPERTY), TimeUnit.MINUTES);
    if (props.containsKey(SOCKET_TIMEOUT_MS_PROPERTY))
        this.setSocketTimeout(props.getInt(SOCKET_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
    if (props.containsKey(SOCKET_KEEPALIVE_PROPERTY))
        this.setSocketKeepAlive(props.getBoolean(SOCKET_KEEPALIVE_PROPERTY));
    if (props.containsKey(SELECTORS_PROPERTY))
        this.setSelectors(props.getInt(SELECTORS_PROPERTY));
    if (props.containsKey(ROUTING_TIMEOUT_MS_PROPERTY))
        this.setRoutingTimeout(props.getInt(ROUTING_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
    // By default, make all the timeouts equal to routing timeout
    timeoutConfig = new TimeoutConfig(routingTimeoutMs, false);
    if (props.containsKey(GETALL_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, props.getInt(GETALL_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(GET_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE, props.getInt(GET_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(PUT_ROUTING_TIMEOUT_MS_PROPERTY)) {
        long putTimeoutMs = props.getInt(PUT_ROUTING_TIMEOUT_MS_PROPERTY);
        timeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, putTimeoutMs);
        // By default, use the same thing for getVersions() also
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, putTimeoutMs);
    }
    // of course, if someone overrides it, we will respect that
    if (props.containsKey(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, props.getInt(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(DELETE_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE, props.getInt(DELETE_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ALLOW_PARTIAL_GETALLS_PROPERTY))
        timeoutConfig.setPartialGetAllAllowed(props.getBoolean(ALLOW_PARTIAL_GETALLS_PROPERTY));
    if (props.containsKey(SOCKET_BUFFER_SIZE_PROPERTY))
        this.setSocketBufferSize(props.getInt(SOCKET_BUFFER_SIZE_PROPERTY));
    if (props.containsKey(SERIALIZER_FACTORY_CLASS_PROPERTY)) {
        Class<?> factoryClass = ReflectUtils.loadClass(props.getString(SERIALIZER_FACTORY_CLASS_PROPERTY));
        SerializerFactory factory = (SerializerFactory) ReflectUtils.callConstructor(factoryClass, new Object[] {});
        this.setSerializerFactory(factory);
    }
    if (props.containsKey(BOOTSTRAP_URLS_PROPERTY))
        this.setBootstrapUrls(props.getList(BOOTSTRAP_URLS_PROPERTY));
    if (props.containsKey(REQUEST_FORMAT_PROPERTY))
        this.setRequestFormatType(RequestFormatType.fromCode(props.getString(REQUEST_FORMAT_PROPERTY)));
    if (props.containsKey(ENABLE_JMX_PROPERTY))
        this.setEnableJmx(props.getBoolean(ENABLE_JMX_PROPERTY));
    if (props.containsKey(ENABLE_LAZY_PROPERTY))
        this.setEnableLazy(props.getBoolean(ENABLE_LAZY_PROPERTY));
    if (props.containsKey(CLIENT_ZONE_ID))
        this.setClientZoneId(props.getInt(CLIENT_ZONE_ID));
    if (props.containsKey(CACHE_STORE_CLIENTS)) {
        this.setCacheStoreClients(props.getBoolean(CACHE_STORE_CLIENTS));
    }
    if (props.containsKey(IDENTIFIER_STRING)) {
        this.setIdentifierString(props.getString(IDENTIFIER_STRING));
    }
    if (props.containsKey(USE_DEFAULT_CLIENT))
        this.enableDefaultClient(props.getBoolean(USE_DEFAULT_CLIENT));
    if (props.containsKey(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY))
        this.setFailureDetectorImplementation(props.getString(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY))
        this.setMaximumTolerableFatalFailures(props.getInt(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY));
    // it over.
    if (props.containsKey(NODE_BANNAGE_MS_PROPERTY) && !props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY)) {
        props.put(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY, props.get(NODE_BANNAGE_MS_PROPERTY));
    }
    if (props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY))
        this.setFailureDetectorBannagePeriod(props.getLong(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_THRESHOLD_PROPERTY))
        this.setFailureDetectorThreshold(props.getInt(FAILUREDETECTOR_THRESHOLD_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY))
        this.setFailureDetectorThresholdCountMinimum(props.getInt(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY))
        this.setFailureDetectorThresholdInterval(props.getLong(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY))
        this.setFailureDetectorAsyncRecoveryInterval(props.getLong(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY))
        this.setFailureDetectorCatastrophicErrorTypes(props.getList(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY))
        this.setFailureDetectorRequestLengthThreshold(props.getLong(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY));
    else
        this.setFailureDetectorRequestLengthThreshold(getSocketTimeout(TimeUnit.MILLISECONDS));
    if (props.containsKey(MAX_BOOTSTRAP_RETRIES))
        this.setMaxBootstrapRetries(props.getInt(MAX_BOOTSTRAP_RETRIES));
    if (props.containsKey(FETCH_ALL_STORES_XML_IN_BOOTSTRAP)) {
        this.setFetchAllStoresXmlInBootstrap(props.getBoolean(FETCH_ALL_STORES_XML_IN_BOOTSTRAP));
    }
    if (props.containsKey(BOOTSTRAP_RETRY_WAIT_TIME_SECONDS)) {
        this.setBootstrapRetryWaitTimeSeconds(props.getInt(BOOTSTRAP_RETRY_WAIT_TIME_SECONDS));
    }
    if (props.containsKey(CLIENT_CONTEXT_NAME)) {
        this.setClientContextName(props.getString(CLIENT_CONTEXT_NAME));
    }
    if (props.containsKey(ASYNC_CHECK_METADATA_INTERVAL)) {
        this.setAsyncMetadataRefreshInMs(props.getLong(ASYNC_CHECK_METADATA_INTERVAL));
    }
    if (props.containsKey(CLIENT_REGISTRY_REFRESH_INTERVAL)) {
        this.setClientRegistryUpdateIntervalInSecs(props.getInt(CLIENT_REGISTRY_REFRESH_INTERVAL));
    }
    if (props.containsKey(ASYNC_JOB_THREAD_POOL_SIZE)) {
        this.setAsyncJobThreadPoolSize(props.getInt(ASYNC_JOB_THREAD_POOL_SIZE));
    }
    /* Check for system store paramaters if any */
    if (props.containsKey(SYS_MAX_CONNECTIONS_PER_NODE)) {
        this.setSysMaxConnectionsPerNode(props.getInt(SYS_MAX_CONNECTIONS_PER_NODE));
    }
    if (props.containsKey(SYS_ROUTING_TIMEOUT_MS)) {
        this.setSysRoutingTimeout(props.getInt(SYS_ROUTING_TIMEOUT_MS));
    }
    if (props.containsKey(SYS_SOCKET_TIMEOUT_MS)) {
        this.setSysSocketTimeout(props.getInt(SYS_SOCKET_TIMEOUT_MS));
    }
    if (props.containsKey(SYS_CONNECTION_TIMEOUT_MS)) {
        this.setSysConnectionTimeout(props.getInt(SYS_CONNECTION_TIMEOUT_MS));
    }
    if (props.containsKey(SYS_ENABLE_JMX)) {
        this.setSysEnableJmx(props.getBoolean(SYS_ENABLE_JMX));
    }
    if (props.containsKey(ENABLE_COMPRESSION_LAYER)) {
        this.setEnableCompressionLayer(props.getBoolean(ENABLE_COMPRESSION_LAYER));
    }
    if (props.containsKey(ENABLE_SERIALIZATION_LAYER)) {
        this.setEnableSerializationLayer(props.getBoolean(ENABLE_SERIALIZATION_LAYER));
    }
    if (props.containsKey(ENABLE_INCONSISTENCY_RESOLVING_LAYER)) {
        this.setEnableInconsistencyResolvingLayer(props.getBoolean(ENABLE_INCONSISTENCY_RESOLVING_LAYER));
    }
    if (props.containsKey(FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY)) {
        this.setFatClientWrapperCorePoolSize(props.getInt(FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY, this.fatClientWrapperCorePoolSize));
    }
    if (props.containsKey(FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY)) {
        this.setFatClientWrapperMaxPoolSize(props.getInt(FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY, this.fatClientWrapperMaxPoolSize));
    }
    if (props.containsKey(FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS)) {
        this.setFatClientWrapperKeepAliveInSecs(props.getInt(FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS, this.fatClientWrapperKeepAliveInSecs));
    }
    if (props.containsKey(GET_OP_ZONE_AFFINITY)) {
        this.getZoneAffinity().setEnableGetOpZoneAffinity(props.getBoolean(GET_OP_ZONE_AFFINITY));
    }
    if (props.containsKey(GETALL_OP_ZONE_AFFINITY)) {
        this.getZoneAffinity().setEnableGetAllOpZoneAffinity(props.getBoolean(GETALL_OP_ZONE_AFFINITY));
    }
    if (props.containsKey(GETVERSIONS_OP_ZONE_AFFINITY)) {
        this.getZoneAffinity().setEnableGetVersionsOpZoneAffinity(props.getBoolean(GETVERSIONS_OP_ZONE_AFFINITY));
    }
    if (props.containsKey(IDENTIFIER_STRING_KEY)) {
        this.setIdentifierString(props.getString(IDENTIFIER_STRING_KEY));
    }
}

Example 5 with SerializerFactory

use of voldemort.serialization.SerializerFactory in project voldemort by voldemort.

the class ViewStorageConfiguration method getStore.

public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    String name = storeDef.getName();
    StoreDefinition def = StoreUtils.getStoreDef(storeDefs, name);
    String targetName = def.getViewTargetStoreName();
    StoreDefinition targetDef = StoreUtils.getStoreDef(storeDefs, targetName);
    StorageEngine<ByteArray, byte[], byte[]> target = storeRepo.getStorageEngine(targetName);
    if (target == null)
        throw new VoldemortException("View \"" + name + "\" has a target store \"" + targetName + "\" which does not exist.");
    String factoryName = def.getSerializerFactory();
    SerializerFactory factory;
    if (factoryName == null)
        factory = new DefaultSerializerFactory();
    else
        factory = loadSerializerFactory(factoryName);
    CompressionStrategy valueCompressionStrategy = null;
    if (targetDef.getValueSerializer().hasCompression()) {
        valueCompressionStrategy = new CompressionStrategyFactory().get(targetDef.getValueSerializer().getCompression());
    }
    View<?, ?, ?, ?> view = loadTransformation(def.getValueTransformation());
    return new ViewStorageEngine(name, target, factory.getSerializer(def.getValueSerializer()), def.getTransformsSerializer() != null ? factory.getSerializer(def.getTransformsSerializer()) : null, factory.getSerializer(targetDef.getKeySerializer()), factory.getSerializer(targetDef.getValueSerializer()), valueCompressionStrategy, view);
}