Search in sources :

Example 16 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class BdbSplitStorageEngineTest method testMultipleEnvironment.

@Test
public void testMultipleEnvironment() {
    // lets use all the default values.
    Props props = new Props();
    props.put("node.id", 1);
    props.put("voldemort.home", "test/common/voldemort/config");
    VoldemortConfig voldemortConfig = new VoldemortConfig(props);
    voldemortConfig.setBdbCacheSize(1 * 1024 * 1024);
    voldemortConfig.setBdbOneEnvPerStore(true);
    voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
    voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId);
    bdbStorage = new BdbStorageConfiguration(voldemortConfig);
    BdbStorageEngine storeA = (BdbStorageEngine) bdbStorage.getStore(TestUtils.makeStoreDefinition("storeA"), TestUtils.makeSingleNodeRoutingStrategy());
    BdbStorageEngine storeB = (BdbStorageEngine) bdbStorage.getStore(TestUtils.makeStoreDefinition("storeB"), TestUtils.makeSingleNodeRoutingStrategy());
    storeA.put(TestUtils.toByteArray("testKey1"), new Versioned<byte[]>("value".getBytes()), null);
    storeA.put(TestUtils.toByteArray("testKey2"), new Versioned<byte[]>("value".getBytes()), null);
    storeA.put(TestUtils.toByteArray("testKey3"), new Versioned<byte[]>("value".getBytes()), null);
    storeB.put(TestUtils.toByteArray("testKey1"), new Versioned<byte[]>("value".getBytes()), null);
    storeB.put(TestUtils.toByteArray("testKey2"), new Versioned<byte[]>("value".getBytes()), null);
    storeB.put(TestUtils.toByteArray("testKey3"), new Versioned<byte[]>("value".getBytes()), null);
    storeA.close();
    storeB.close();
    assertEquals("StoreA BDB file should exists.", true, (new File(bdbMasterDir + "/" + "storeA").exists()));
    assertEquals("StoreB BDB file should  exists.", true, (new File(bdbMasterDir + "/" + "storeB").exists()));
}
Also used : Props(voldemort.utils.Props) File(java.io.File) VoldemortConfig(voldemort.server.VoldemortConfig) Test(org.junit.Test)

Example 17 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class ClientConfig method setProperties.

private void setProperties(Properties properties) {
    Props props = new Props(properties);
    if (props.containsKey(MAX_CONNECTIONS_PER_NODE_PROPERTY))
        this.setMaxConnectionsPerNode(props.getInt(MAX_CONNECTIONS_PER_NODE_PROPERTY));
    if (props.containsKey(MAX_TOTAL_CONNECTIONS_PROPERTY))
        this.setMaxTotalConnections(props.getInt(MAX_TOTAL_CONNECTIONS_PROPERTY));
    if (props.containsKey(MAX_THREADS_PROPERTY))
        this.setMaxThreads(props.getInt(MAX_THREADS_PROPERTY));
    if (props.containsKey(MAX_QUEUED_REQUESTS_PROPERTY))
        this.setMaxQueuedRequests(props.getInt(MAX_QUEUED_REQUESTS_PROPERTY));
    if (props.containsKey(THREAD_IDLE_MS_PROPERTY))
        this.setThreadIdleTime(props.getLong(THREAD_IDLE_MS_PROPERTY), TimeUnit.MILLISECONDS);
    if (props.containsKey(CONNECTION_TIMEOUT_MS_PROPERTY))
        this.setConnectionTimeout(props.getInt(CONNECTION_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
    if (props.containsKey(IDLE_CONNECTION_TIMEOUT_MINUTES_PROPERTY))
        this.setIdleConnectionTimeout(props.getInt(IDLE_CONNECTION_TIMEOUT_MINUTES_PROPERTY), TimeUnit.MINUTES);
    if (props.containsKey(SOCKET_TIMEOUT_MS_PROPERTY))
        this.setSocketTimeout(props.getInt(SOCKET_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
    if (props.containsKey(SOCKET_KEEPALIVE_PROPERTY))
        this.setSocketKeepAlive(props.getBoolean(SOCKET_KEEPALIVE_PROPERTY));
    if (props.containsKey(SELECTORS_PROPERTY))
        this.setSelectors(props.getInt(SELECTORS_PROPERTY));
    if (props.containsKey(ROUTING_TIMEOUT_MS_PROPERTY))
        this.setRoutingTimeout(props.getInt(ROUTING_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
    // By default, make all the timeouts equal to routing timeout
    timeoutConfig = new TimeoutConfig(routingTimeoutMs, false);
    if (props.containsKey(GETALL_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, props.getInt(GETALL_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(GET_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE, props.getInt(GET_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(PUT_ROUTING_TIMEOUT_MS_PROPERTY)) {
        long putTimeoutMs = props.getInt(PUT_ROUTING_TIMEOUT_MS_PROPERTY);
        timeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, putTimeoutMs);
        // By default, use the same thing for getVersions() also
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, putTimeoutMs);
    }
    // of course, if someone overrides it, we will respect that
    if (props.containsKey(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, props.getInt(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(DELETE_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE, props.getInt(DELETE_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ALLOW_PARTIAL_GETALLS_PROPERTY))
        timeoutConfig.setPartialGetAllAllowed(props.getBoolean(ALLOW_PARTIAL_GETALLS_PROPERTY));
    if (props.containsKey(SOCKET_BUFFER_SIZE_PROPERTY))
        this.setSocketBufferSize(props.getInt(SOCKET_BUFFER_SIZE_PROPERTY));
    if (props.containsKey(SERIALIZER_FACTORY_CLASS_PROPERTY)) {
        Class<?> factoryClass = ReflectUtils.loadClass(props.getString(SERIALIZER_FACTORY_CLASS_PROPERTY));
        SerializerFactory factory = (SerializerFactory) ReflectUtils.callConstructor(factoryClass, new Object[] {});
        this.setSerializerFactory(factory);
    }
    if (props.containsKey(BOOTSTRAP_URLS_PROPERTY))
        this.setBootstrapUrls(props.getList(BOOTSTRAP_URLS_PROPERTY));
    if (props.containsKey(REQUEST_FORMAT_PROPERTY))
        this.setRequestFormatType(RequestFormatType.fromCode(props.getString(REQUEST_FORMAT_PROPERTY)));
    if (props.containsKey(ENABLE_JMX_PROPERTY))
        this.setEnableJmx(props.getBoolean(ENABLE_JMX_PROPERTY));
    if (props.containsKey(ENABLE_LAZY_PROPERTY))
        this.setEnableLazy(props.getBoolean(ENABLE_LAZY_PROPERTY));
    if (props.containsKey(CLIENT_ZONE_ID))
        this.setClientZoneId(props.getInt(CLIENT_ZONE_ID));
    if (props.containsKey(CACHE_STORE_CLIENTS)) {
        this.setCacheStoreClients(props.getBoolean(CACHE_STORE_CLIENTS));
    }
    if (props.containsKey(IDENTIFIER_STRING)) {
        this.setIdentifierString(props.getString(IDENTIFIER_STRING));
    }
    if (props.containsKey(USE_DEFAULT_CLIENT))
        this.enableDefaultClient(props.getBoolean(USE_DEFAULT_CLIENT));
    if (props.containsKey(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY))
        this.setFailureDetectorImplementation(props.getString(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY))
        this.setMaximumTolerableFatalFailures(props.getInt(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY));
    // it over.
    if (props.containsKey(NODE_BANNAGE_MS_PROPERTY) && !props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY)) {
        props.put(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY, props.get(NODE_BANNAGE_MS_PROPERTY));
    }
    if (props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY))
        this.setFailureDetectorBannagePeriod(props.getLong(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_THRESHOLD_PROPERTY))
        this.setFailureDetectorThreshold(props.getInt(FAILUREDETECTOR_THRESHOLD_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY))
        this.setFailureDetectorThresholdCountMinimum(props.getInt(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY))
        this.setFailureDetectorThresholdInterval(props.getLong(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY))
        this.setFailureDetectorAsyncRecoveryInterval(props.getLong(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY))
        this.setFailureDetectorCatastrophicErrorTypes(props.getList(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY));
    if (props.containsKey(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY))
        this.setFailureDetectorRequestLengthThreshold(props.getLong(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY));
    else
        this.setFailureDetectorRequestLengthThreshold(getSocketTimeout(TimeUnit.MILLISECONDS));
    if (props.containsKey(MAX_BOOTSTRAP_RETRIES))
        this.setMaxBootstrapRetries(props.getInt(MAX_BOOTSTRAP_RETRIES));
    if (props.containsKey(FETCH_ALL_STORES_XML_IN_BOOTSTRAP)) {
        this.setFetchAllStoresXmlInBootstrap(props.getBoolean(FETCH_ALL_STORES_XML_IN_BOOTSTRAP));
    }
    if (props.containsKey(BOOTSTRAP_RETRY_WAIT_TIME_SECONDS)) {
        this.setBootstrapRetryWaitTimeSeconds(props.getInt(BOOTSTRAP_RETRY_WAIT_TIME_SECONDS));
    }
    if (props.containsKey(CLIENT_CONTEXT_NAME)) {
        this.setClientContextName(props.getString(CLIENT_CONTEXT_NAME));
    }
    if (props.containsKey(ASYNC_CHECK_METADATA_INTERVAL)) {
        this.setAsyncMetadataRefreshInMs(props.getLong(ASYNC_CHECK_METADATA_INTERVAL));
    }
    if (props.containsKey(CLIENT_REGISTRY_REFRESH_INTERVAL)) {
        this.setClientRegistryUpdateIntervalInSecs(props.getInt(CLIENT_REGISTRY_REFRESH_INTERVAL));
    }
    if (props.containsKey(ASYNC_JOB_THREAD_POOL_SIZE)) {
        this.setAsyncJobThreadPoolSize(props.getInt(ASYNC_JOB_THREAD_POOL_SIZE));
    }
    /* Check for system store paramaters if any */
    if (props.containsKey(SYS_MAX_CONNECTIONS_PER_NODE)) {
        this.setSysMaxConnectionsPerNode(props.getInt(SYS_MAX_CONNECTIONS_PER_NODE));
    }
    if (props.containsKey(SYS_ROUTING_TIMEOUT_MS)) {
        this.setSysRoutingTimeout(props.getInt(SYS_ROUTING_TIMEOUT_MS));
    }
    if (props.containsKey(SYS_SOCKET_TIMEOUT_MS)) {
        this.setSysSocketTimeout(props.getInt(SYS_SOCKET_TIMEOUT_MS));
    }
    if (props.containsKey(SYS_CONNECTION_TIMEOUT_MS)) {
        this.setSysConnectionTimeout(props.getInt(SYS_CONNECTION_TIMEOUT_MS));
    }
    if (props.containsKey(SYS_ENABLE_JMX)) {
        this.setSysEnableJmx(props.getBoolean(SYS_ENABLE_JMX));
    }
    if (props.containsKey(ENABLE_COMPRESSION_LAYER)) {
        this.setEnableCompressionLayer(props.getBoolean(ENABLE_COMPRESSION_LAYER));
    }
    if (props.containsKey(ENABLE_SERIALIZATION_LAYER)) {
        this.setEnableSerializationLayer(props.getBoolean(ENABLE_SERIALIZATION_LAYER));
    }
    if (props.containsKey(ENABLE_INCONSISTENCY_RESOLVING_LAYER)) {
        this.setEnableInconsistencyResolvingLayer(props.getBoolean(ENABLE_INCONSISTENCY_RESOLVING_LAYER));
    }
    if (props.containsKey(FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY)) {
        this.setFatClientWrapperCorePoolSize(props.getInt(FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY, this.fatClientWrapperCorePoolSize));
    }
    if (props.containsKey(FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY)) {
        this.setFatClientWrapperMaxPoolSize(props.getInt(FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY, this.fatClientWrapperMaxPoolSize));
    }
    if (props.containsKey(FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS)) {
        this.setFatClientWrapperKeepAliveInSecs(props.getInt(FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS, this.fatClientWrapperKeepAliveInSecs));
    }
    if (props.containsKey(GET_OP_ZONE_AFFINITY)) {
        this.getZoneAffinity().setEnableGetOpZoneAffinity(props.getBoolean(GET_OP_ZONE_AFFINITY));
    }
    if (props.containsKey(GETALL_OP_ZONE_AFFINITY)) {
        this.getZoneAffinity().setEnableGetAllOpZoneAffinity(props.getBoolean(GETALL_OP_ZONE_AFFINITY));
    }
    if (props.containsKey(GETVERSIONS_OP_ZONE_AFFINITY)) {
        this.getZoneAffinity().setEnableGetVersionsOpZoneAffinity(props.getBoolean(GETVERSIONS_OP_ZONE_AFFINITY));
    }
    if (props.containsKey(IDENTIFIER_STRING_KEY)) {
        this.setIdentifierString(props.getString(IDENTIFIER_STRING_KEY));
    }
}
Also used : DefaultSerializerFactory(voldemort.serialization.DefaultSerializerFactory) SerializerFactory(voldemort.serialization.SerializerFactory) Props(voldemort.utils.Props)

Example 18 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class BlockingSlopPusherTest method setUp.

@Override
protected void setUp() throws Exception {
    Cluster cluster = makeCluster(3);
    repo = new StoreRepository();
    repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
    repo.addNodeStore(0, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
    repo.addNodeStore(2, new FailingStore<ByteArray, byte[], byte[]>(STORE_NAME));
    MetadataStore metadataStore = ServerTestUtils.createMetadataStore(cluster, ServerTestUtils.getStoreDefs(1));
    Props props = new Props();
    props.put("node.id", 0);
    props.put("voldemort.home", "/");
    pusher = new BlockingSlopPusherJob(repo, metadataStore, new NoopFailureDetector(), new VoldemortConfig(props), new ScanPermitWrapper(1));
}
Also used : BlockingSlopPusherJob(voldemort.server.scheduler.slop.BlockingSlopPusherJob) NoopFailureDetector(voldemort.cluster.failuredetector.NoopFailureDetector) ScanPermitWrapper(voldemort.server.storage.ScanPermitWrapper) Cluster(voldemort.cluster.Cluster) StoreRepository(voldemort.server.StoreRepository) Props(voldemort.utils.Props) VoldemortConfig(voldemort.server.VoldemortConfig) MetadataStore(voldemort.store.metadata.MetadataStore) InMemoryStorageEngine(voldemort.store.memory.InMemoryStorageEngine) ByteArray(voldemort.utils.ByteArray) SlopStorageEngine(voldemort.store.slop.SlopStorageEngine)

Example 19 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class StoreVersionManager method removeVersion.

private void removeVersion(long version, boolean alsoSyncRemoteState) {
    if (currentVersion == version) {
        // Should we throw instead?
        currentVersion = -1;
    }
    versionToEnabledMap.remove(version);
    if (alsoSyncRemoteState && config != null && config.getHighAvailabilityStateAutoCleanUp()) {
        FailedFetchLock failedFetchLock = null;
        try {
            failedFetchLock = FailedFetchLock.getLock(config, new Props());
            removeRemoteObsoleteState(failedFetchLock);
        } catch (Exception e) {
            logger.error("Failed to execute failedFetchLock.removeObsoleteStateForStore() for store " + storeName, e);
        } finally {
            IOUtils.closeQuietly(failedFetchLock);
        }
    }
}
Also used : FailedFetchLock(voldemort.store.readonly.swapper.FailedFetchLock) Props(voldemort.utils.Props) IOException(java.io.IOException) PersistenceFailureException(voldemort.store.PersistenceFailureException)

Example 20 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class HadoopStoreBuilderTest method testHadoopBuild.

@Test
public void testHadoopBuild() throws Exception {
    // create test data
    Map<String, String> values = new HashMap<String, String>();
    File testDir = TestUtils.createTempDir();
    File tempDir = new File(testDir, "temp"), tempDir2 = new File(testDir, "temp2");
    File outputDir = new File(testDir, "output"), outputDir2 = new File(testDir, "output2");
    File storeDir = TestUtils.createTempDir(testDir);
    for (int i = 0; i < 200; i++) values.put(Integer.toString(i), Integer.toBinaryString(i));
    // write test data to text file
    File inputFile = File.createTempFile("input", ".txt", testDir);
    inputFile.deleteOnExit();
    StringBuilder contents = new StringBuilder();
    for (Map.Entry<String, String> entry : values.entrySet()) contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
    FileUtils.writeStringToFile(inputFile, contents.toString());
    String storeName = "test";
    SerializerDefinition serDef = new SerializerDefinition("string");
    Cluster cluster = ServerTestUtils.getLocalCluster(1);
    // Test backwards compatibility
    StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    HadoopStoreBuilder builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir2.getAbsolutePath()), new Path(outputDir2.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
    builder.build();
    // Check if checkSum is generated in outputDir
    File nodeFile = new File(outputDir, "node-0");
    // Check if metadata file exists
    File metadataFile = new File(nodeFile, ".metadata");
    Assert.assertTrue("Metadata file should exist!", metadataFile.exists());
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata(metadataFile);
    if (saveKeys)
        Assert.assertEquals("In saveKeys mode, the metadata format should be READONLY_V2!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V2.getCode());
    else
        Assert.assertEquals("In legacy mode (saveKeys==false), the metadata format should be READONLY_V1!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V1.getCode());
    Assert.assertEquals("Checksum type should be MD5!", metadata.get(ReadOnlyStorageMetadata.CHECKSUM_TYPE), CheckSum.toString(CheckSumType.MD5));
    // Check contents of checkSum file
    byte[] md5 = Hex.decodeHex(((String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM)).toCharArray());
    byte[] checkSumBytes = CheckSumTests.calculateCheckSum(nodeFile.listFiles(), CheckSumType.MD5);
    Assert.assertEquals("Checksum is not as excepted!", 0, ByteUtils.compare(checkSumBytes, md5));
    // check if fetching works
    HdfsFetcher fetcher = new HdfsFetcher();
    // Fetch to version directory
    File versionDir = new File(storeDir, "version-0");
    fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
    Assert.assertTrue("Version directory should exist!", versionDir.exists());
    // open store
    @SuppressWarnings("unchecked") Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
    ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, searchStrategy, new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
    Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
    // check values
    for (Map.Entry<String, String> entry : values.entrySet()) {
        String key = entry.getKey();
        try {
            List<Versioned<Object>> found = store.get(key, null);
            Assert.assertEquals("Incorrect number of results", 1, found.size());
            Assert.assertEquals(entry.getValue(), found.get(0).getValue());
        } catch (VoldemortException e) {
            throw new VoldemortException("Got an exception while trying to get key '" + key + "'.", e);
        }
    }
    // also check the iterator - first key iterator...
    try {
        ClosableIterator<ByteArray> keyIterator = engine.keys();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (keyIterator.hasNext()) {
            Assert.assertTrue(values.containsKey(serializer.toObject(keyIterator.next().get())));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
    // ... and entry iterator
    try {
        ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
        if (!saveKeys) {
            fail("Should have thrown an exception since this RO format does not support iterators");
        }
        int numElements = 0;
        while (entryIterator.hasNext()) {
            Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
            Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
            numElements++;
        }
        Assert.assertEquals(numElements, values.size());
    } catch (UnsupportedOperationException e) {
        if (saveKeys) {
            fail("Should not have thrown an exception since this RO format does support iterators");
        }
    }
}
Also used : Versioned(voldemort.versioning.Versioned) HashMap(java.util.HashMap) RoutingStrategyFactory(voldemort.routing.RoutingStrategyFactory) Props(voldemort.utils.Props) VoldemortException(voldemort.VoldemortException) ReadOnlyStorageMetadata(voldemort.store.readonly.ReadOnlyStorageMetadata) StoreDefinition(voldemort.store.StoreDefinition) ByteArray(voldemort.utils.ByteArray) JobConf(org.apache.hadoop.mapred.JobConf) Serializer(voldemort.serialization.Serializer) Pair(voldemort.utils.Pair) StoreDefinitionBuilder(voldemort.store.StoreDefinitionBuilder) Path(org.apache.hadoop.fs.Path) Cluster(voldemort.cluster.Cluster) ReadOnlyStorageEngine(voldemort.store.readonly.ReadOnlyStorageEngine) DefaultSerializerFactory(voldemort.serialization.DefaultSerializerFactory) HdfsFetcher(voldemort.store.readonly.fetcher.HdfsFetcher) TextInputFormat(org.apache.hadoop.mapred.TextInputFormat) File(java.io.File) Map(java.util.Map) HashMap(java.util.HashMap) SerializerDefinition(voldemort.serialization.SerializerDefinition) Test(org.junit.Test)

Aggregations

Props (voldemort.utils.Props)33 VoldemortConfig (voldemort.server.VoldemortConfig)14 File (java.io.File)13 StoreDefinition (voldemort.store.StoreDefinition)10 Test (org.junit.Test)9 ByteArray (voldemort.utils.ByteArray)9 IOException (java.io.IOException)8 Path (org.apache.hadoop.fs.Path)6 Cluster (voldemort.cluster.Cluster)6 JobConf (org.apache.hadoop.mapred.JobConf)4 VoldemortException (voldemort.VoldemortException)4 BdbStorageConfiguration (voldemort.store.bdb.BdbStorageConfiguration)4 ObsoleteVersionException (voldemort.versioning.ObsoleteVersionException)4 HashMap (java.util.HashMap)3 Map (java.util.Map)3 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)3 OptionParser (joptsimple.OptionParser)3 OptionSet (joptsimple.OptionSet)3 Before (org.junit.Before)3 DefaultSerializerFactory (voldemort.serialization.DefaultSerializerFactory)3