use of voldemort.utils.Props in project voldemort by voldemort.
the class BdbSplitStorageEngineTest method testMultipleEnvironment.
@Test
public void testMultipleEnvironment() {
// lets use all the default values.
Props props = new Props();
props.put("node.id", 1);
props.put("voldemort.home", "test/common/voldemort/config");
VoldemortConfig voldemortConfig = new VoldemortConfig(props);
voldemortConfig.setBdbCacheSize(1 * 1024 * 1024);
voldemortConfig.setBdbOneEnvPerStore(true);
voldemortConfig.setBdbDataDirectory(bdbMasterDir.toURI().getPath());
voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId);
bdbStorage = new BdbStorageConfiguration(voldemortConfig);
BdbStorageEngine storeA = (BdbStorageEngine) bdbStorage.getStore(TestUtils.makeStoreDefinition("storeA"), TestUtils.makeSingleNodeRoutingStrategy());
BdbStorageEngine storeB = (BdbStorageEngine) bdbStorage.getStore(TestUtils.makeStoreDefinition("storeB"), TestUtils.makeSingleNodeRoutingStrategy());
storeA.put(TestUtils.toByteArray("testKey1"), new Versioned<byte[]>("value".getBytes()), null);
storeA.put(TestUtils.toByteArray("testKey2"), new Versioned<byte[]>("value".getBytes()), null);
storeA.put(TestUtils.toByteArray("testKey3"), new Versioned<byte[]>("value".getBytes()), null);
storeB.put(TestUtils.toByteArray("testKey1"), new Versioned<byte[]>("value".getBytes()), null);
storeB.put(TestUtils.toByteArray("testKey2"), new Versioned<byte[]>("value".getBytes()), null);
storeB.put(TestUtils.toByteArray("testKey3"), new Versioned<byte[]>("value".getBytes()), null);
storeA.close();
storeB.close();
assertEquals("StoreA BDB file should exists.", true, (new File(bdbMasterDir + "/" + "storeA").exists()));
assertEquals("StoreB BDB file should exists.", true, (new File(bdbMasterDir + "/" + "storeB").exists()));
}
use of voldemort.utils.Props in project voldemort by voldemort.
the class ClientConfig method setProperties.
private void setProperties(Properties properties) {
Props props = new Props(properties);
if (props.containsKey(MAX_CONNECTIONS_PER_NODE_PROPERTY))
this.setMaxConnectionsPerNode(props.getInt(MAX_CONNECTIONS_PER_NODE_PROPERTY));
if (props.containsKey(MAX_TOTAL_CONNECTIONS_PROPERTY))
this.setMaxTotalConnections(props.getInt(MAX_TOTAL_CONNECTIONS_PROPERTY));
if (props.containsKey(MAX_THREADS_PROPERTY))
this.setMaxThreads(props.getInt(MAX_THREADS_PROPERTY));
if (props.containsKey(MAX_QUEUED_REQUESTS_PROPERTY))
this.setMaxQueuedRequests(props.getInt(MAX_QUEUED_REQUESTS_PROPERTY));
if (props.containsKey(THREAD_IDLE_MS_PROPERTY))
this.setThreadIdleTime(props.getLong(THREAD_IDLE_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(CONNECTION_TIMEOUT_MS_PROPERTY))
this.setConnectionTimeout(props.getInt(CONNECTION_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(IDLE_CONNECTION_TIMEOUT_MINUTES_PROPERTY))
this.setIdleConnectionTimeout(props.getInt(IDLE_CONNECTION_TIMEOUT_MINUTES_PROPERTY), TimeUnit.MINUTES);
if (props.containsKey(SOCKET_TIMEOUT_MS_PROPERTY))
this.setSocketTimeout(props.getInt(SOCKET_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
if (props.containsKey(SOCKET_KEEPALIVE_PROPERTY))
this.setSocketKeepAlive(props.getBoolean(SOCKET_KEEPALIVE_PROPERTY));
if (props.containsKey(SELECTORS_PROPERTY))
this.setSelectors(props.getInt(SELECTORS_PROPERTY));
if (props.containsKey(ROUTING_TIMEOUT_MS_PROPERTY))
this.setRoutingTimeout(props.getInt(ROUTING_TIMEOUT_MS_PROPERTY), TimeUnit.MILLISECONDS);
// By default, make all the timeouts equal to routing timeout
timeoutConfig = new TimeoutConfig(routingTimeoutMs, false);
if (props.containsKey(GETALL_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, props.getInt(GETALL_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(GET_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE, props.getInt(GET_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(PUT_ROUTING_TIMEOUT_MS_PROPERTY)) {
long putTimeoutMs = props.getInt(PUT_ROUTING_TIMEOUT_MS_PROPERTY);
timeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, putTimeoutMs);
// By default, use the same thing for getVersions() also
timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, putTimeoutMs);
}
// of course, if someone overrides it, we will respect that
if (props.containsKey(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, props.getInt(GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(DELETE_ROUTING_TIMEOUT_MS_PROPERTY))
timeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE, props.getInt(DELETE_ROUTING_TIMEOUT_MS_PROPERTY));
if (props.containsKey(ALLOW_PARTIAL_GETALLS_PROPERTY))
timeoutConfig.setPartialGetAllAllowed(props.getBoolean(ALLOW_PARTIAL_GETALLS_PROPERTY));
if (props.containsKey(SOCKET_BUFFER_SIZE_PROPERTY))
this.setSocketBufferSize(props.getInt(SOCKET_BUFFER_SIZE_PROPERTY));
if (props.containsKey(SERIALIZER_FACTORY_CLASS_PROPERTY)) {
Class<?> factoryClass = ReflectUtils.loadClass(props.getString(SERIALIZER_FACTORY_CLASS_PROPERTY));
SerializerFactory factory = (SerializerFactory) ReflectUtils.callConstructor(factoryClass, new Object[] {});
this.setSerializerFactory(factory);
}
if (props.containsKey(BOOTSTRAP_URLS_PROPERTY))
this.setBootstrapUrls(props.getList(BOOTSTRAP_URLS_PROPERTY));
if (props.containsKey(REQUEST_FORMAT_PROPERTY))
this.setRequestFormatType(RequestFormatType.fromCode(props.getString(REQUEST_FORMAT_PROPERTY)));
if (props.containsKey(ENABLE_JMX_PROPERTY))
this.setEnableJmx(props.getBoolean(ENABLE_JMX_PROPERTY));
if (props.containsKey(ENABLE_LAZY_PROPERTY))
this.setEnableLazy(props.getBoolean(ENABLE_LAZY_PROPERTY));
if (props.containsKey(CLIENT_ZONE_ID))
this.setClientZoneId(props.getInt(CLIENT_ZONE_ID));
if (props.containsKey(CACHE_STORE_CLIENTS)) {
this.setCacheStoreClients(props.getBoolean(CACHE_STORE_CLIENTS));
}
if (props.containsKey(IDENTIFIER_STRING)) {
this.setIdentifierString(props.getString(IDENTIFIER_STRING));
}
if (props.containsKey(USE_DEFAULT_CLIENT))
this.enableDefaultClient(props.getBoolean(USE_DEFAULT_CLIENT));
if (props.containsKey(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY))
this.setFailureDetectorImplementation(props.getString(FAILUREDETECTOR_IMPLEMENTATION_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY))
this.setMaximumTolerableFatalFailures(props.getInt(FAILUREDETECTOR_MAX_TOLERABLE_FATALITIES_PROPERTY));
// it over.
if (props.containsKey(NODE_BANNAGE_MS_PROPERTY) && !props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY)) {
props.put(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY, props.get(NODE_BANNAGE_MS_PROPERTY));
}
if (props.containsKey(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY))
this.setFailureDetectorBannagePeriod(props.getLong(FAILUREDETECTOR_BANNAGE_PERIOD_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_THRESHOLD_PROPERTY))
this.setFailureDetectorThreshold(props.getInt(FAILUREDETECTOR_THRESHOLD_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY))
this.setFailureDetectorThresholdCountMinimum(props.getInt(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY))
this.setFailureDetectorThresholdInterval(props.getLong(FAILUREDETECTOR_THRESHOLD_INTERVAL_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY))
this.setFailureDetectorAsyncRecoveryInterval(props.getLong(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY))
this.setFailureDetectorCatastrophicErrorTypes(props.getList(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES_PROPERTY));
if (props.containsKey(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY))
this.setFailureDetectorRequestLengthThreshold(props.getLong(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD_PROPERTY));
else
this.setFailureDetectorRequestLengthThreshold(getSocketTimeout(TimeUnit.MILLISECONDS));
if (props.containsKey(MAX_BOOTSTRAP_RETRIES))
this.setMaxBootstrapRetries(props.getInt(MAX_BOOTSTRAP_RETRIES));
if (props.containsKey(FETCH_ALL_STORES_XML_IN_BOOTSTRAP)) {
this.setFetchAllStoresXmlInBootstrap(props.getBoolean(FETCH_ALL_STORES_XML_IN_BOOTSTRAP));
}
if (props.containsKey(BOOTSTRAP_RETRY_WAIT_TIME_SECONDS)) {
this.setBootstrapRetryWaitTimeSeconds(props.getInt(BOOTSTRAP_RETRY_WAIT_TIME_SECONDS));
}
if (props.containsKey(CLIENT_CONTEXT_NAME)) {
this.setClientContextName(props.getString(CLIENT_CONTEXT_NAME));
}
if (props.containsKey(ASYNC_CHECK_METADATA_INTERVAL)) {
this.setAsyncMetadataRefreshInMs(props.getLong(ASYNC_CHECK_METADATA_INTERVAL));
}
if (props.containsKey(CLIENT_REGISTRY_REFRESH_INTERVAL)) {
this.setClientRegistryUpdateIntervalInSecs(props.getInt(CLIENT_REGISTRY_REFRESH_INTERVAL));
}
if (props.containsKey(ASYNC_JOB_THREAD_POOL_SIZE)) {
this.setAsyncJobThreadPoolSize(props.getInt(ASYNC_JOB_THREAD_POOL_SIZE));
}
/* Check for system store paramaters if any */
if (props.containsKey(SYS_MAX_CONNECTIONS_PER_NODE)) {
this.setSysMaxConnectionsPerNode(props.getInt(SYS_MAX_CONNECTIONS_PER_NODE));
}
if (props.containsKey(SYS_ROUTING_TIMEOUT_MS)) {
this.setSysRoutingTimeout(props.getInt(SYS_ROUTING_TIMEOUT_MS));
}
if (props.containsKey(SYS_SOCKET_TIMEOUT_MS)) {
this.setSysSocketTimeout(props.getInt(SYS_SOCKET_TIMEOUT_MS));
}
if (props.containsKey(SYS_CONNECTION_TIMEOUT_MS)) {
this.setSysConnectionTimeout(props.getInt(SYS_CONNECTION_TIMEOUT_MS));
}
if (props.containsKey(SYS_ENABLE_JMX)) {
this.setSysEnableJmx(props.getBoolean(SYS_ENABLE_JMX));
}
if (props.containsKey(ENABLE_COMPRESSION_LAYER)) {
this.setEnableCompressionLayer(props.getBoolean(ENABLE_COMPRESSION_LAYER));
}
if (props.containsKey(ENABLE_SERIALIZATION_LAYER)) {
this.setEnableSerializationLayer(props.getBoolean(ENABLE_SERIALIZATION_LAYER));
}
if (props.containsKey(ENABLE_INCONSISTENCY_RESOLVING_LAYER)) {
this.setEnableInconsistencyResolvingLayer(props.getBoolean(ENABLE_INCONSISTENCY_RESOLVING_LAYER));
}
if (props.containsKey(FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY)) {
this.setFatClientWrapperCorePoolSize(props.getInt(FAT_CLIENT_WRAPPER_CORE_POOL_SIZE_PROPERTY, this.fatClientWrapperCorePoolSize));
}
if (props.containsKey(FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY)) {
this.setFatClientWrapperMaxPoolSize(props.getInt(FAT_CLIENT_WRAPPER_MAX_POOL_SIZE_PROPERTY, this.fatClientWrapperMaxPoolSize));
}
if (props.containsKey(FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS)) {
this.setFatClientWrapperKeepAliveInSecs(props.getInt(FAT_CLIENT_WRAPPER_POOL_KEEPALIVE_IN_SECS, this.fatClientWrapperKeepAliveInSecs));
}
if (props.containsKey(GET_OP_ZONE_AFFINITY)) {
this.getZoneAffinity().setEnableGetOpZoneAffinity(props.getBoolean(GET_OP_ZONE_AFFINITY));
}
if (props.containsKey(GETALL_OP_ZONE_AFFINITY)) {
this.getZoneAffinity().setEnableGetAllOpZoneAffinity(props.getBoolean(GETALL_OP_ZONE_AFFINITY));
}
if (props.containsKey(GETVERSIONS_OP_ZONE_AFFINITY)) {
this.getZoneAffinity().setEnableGetVersionsOpZoneAffinity(props.getBoolean(GETVERSIONS_OP_ZONE_AFFINITY));
}
if (props.containsKey(IDENTIFIER_STRING_KEY)) {
this.setIdentifierString(props.getString(IDENTIFIER_STRING_KEY));
}
}
use of voldemort.utils.Props in project voldemort by voldemort.
the class BlockingSlopPusherTest method setUp.
@Override
protected void setUp() throws Exception {
Cluster cluster = makeCluster(3);
repo = new StoreRepository();
repo.setSlopStore(new SlopStorageEngine(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("slop"), cluster));
repo.addNodeStore(0, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
repo.addNodeStore(1, new InMemoryStorageEngine<ByteArray, byte[], byte[]>(STORE_NAME));
repo.addNodeStore(2, new FailingStore<ByteArray, byte[], byte[]>(STORE_NAME));
MetadataStore metadataStore = ServerTestUtils.createMetadataStore(cluster, ServerTestUtils.getStoreDefs(1));
Props props = new Props();
props.put("node.id", 0);
props.put("voldemort.home", "/");
pusher = new BlockingSlopPusherJob(repo, metadataStore, new NoopFailureDetector(), new VoldemortConfig(props), new ScanPermitWrapper(1));
}
use of voldemort.utils.Props in project voldemort by voldemort.
the class StoreVersionManager method removeVersion.
private void removeVersion(long version, boolean alsoSyncRemoteState) {
if (currentVersion == version) {
// Should we throw instead?
currentVersion = -1;
}
versionToEnabledMap.remove(version);
if (alsoSyncRemoteState && config != null && config.getHighAvailabilityStateAutoCleanUp()) {
FailedFetchLock failedFetchLock = null;
try {
failedFetchLock = FailedFetchLock.getLock(config, new Props());
removeRemoteObsoleteState(failedFetchLock);
} catch (Exception e) {
logger.error("Failed to execute failedFetchLock.removeObsoleteStateForStore() for store " + storeName, e);
} finally {
IOUtils.closeQuietly(failedFetchLock);
}
}
}
use of voldemort.utils.Props in project voldemort by voldemort.
the class HadoopStoreBuilderTest method testHadoopBuild.
@Test
public void testHadoopBuild() throws Exception {
// create test data
Map<String, String> values = new HashMap<String, String>();
File testDir = TestUtils.createTempDir();
File tempDir = new File(testDir, "temp"), tempDir2 = new File(testDir, "temp2");
File outputDir = new File(testDir, "output"), outputDir2 = new File(testDir, "output2");
File storeDir = TestUtils.createTempDir(testDir);
for (int i = 0; i < 200; i++) values.put(Integer.toString(i), Integer.toBinaryString(i));
// write test data to text file
File inputFile = File.createTempFile("input", ".txt", testDir);
inputFile.deleteOnExit();
StringBuilder contents = new StringBuilder();
for (Map.Entry<String, String> entry : values.entrySet()) contents.append(entry.getKey() + "\t" + entry.getValue() + "\n");
FileUtils.writeStringToFile(inputFile, contents.toString());
String storeName = "test";
SerializerDefinition serDef = new SerializerDefinition("string");
Cluster cluster = ServerTestUtils.getLocalCluster(1);
// Test backwards compatibility
StoreDefinition def = new StoreDefinitionBuilder().setName(storeName).setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(serDef).setValueSerializer(serDef).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
HadoopStoreBuilder builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir2.getAbsolutePath()), new Path(outputDir2.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
builder.build();
builder = new HadoopStoreBuilder("testHadoopBuild", new Props(), new JobConf(), TextStoreMapper.class, TextInputFormat.class, cluster, def, new Path(tempDir.getAbsolutePath()), new Path(outputDir.getAbsolutePath()), new Path(inputFile.getAbsolutePath()), CheckSumType.MD5, saveKeys, false, 64 * 1024, false, null, false);
builder.build();
// Check if checkSum is generated in outputDir
File nodeFile = new File(outputDir, "node-0");
// Check if metadata file exists
File metadataFile = new File(nodeFile, ".metadata");
Assert.assertTrue("Metadata file should exist!", metadataFile.exists());
ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata(metadataFile);
if (saveKeys)
Assert.assertEquals("In saveKeys mode, the metadata format should be READONLY_V2!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V2.getCode());
else
Assert.assertEquals("In legacy mode (saveKeys==false), the metadata format should be READONLY_V1!", metadata.get(ReadOnlyStorageMetadata.FORMAT), ReadOnlyStorageFormat.READONLY_V1.getCode());
Assert.assertEquals("Checksum type should be MD5!", metadata.get(ReadOnlyStorageMetadata.CHECKSUM_TYPE), CheckSum.toString(CheckSumType.MD5));
// Check contents of checkSum file
byte[] md5 = Hex.decodeHex(((String) metadata.get(ReadOnlyStorageMetadata.CHECKSUM)).toCharArray());
byte[] checkSumBytes = CheckSumTests.calculateCheckSum(nodeFile.listFiles(), CheckSumType.MD5);
Assert.assertEquals("Checksum is not as excepted!", 0, ByteUtils.compare(checkSumBytes, md5));
// check if fetching works
HdfsFetcher fetcher = new HdfsFetcher();
// Fetch to version directory
File versionDir = new File(storeDir, "version-0");
fetcher.fetch(nodeFile.getAbsolutePath(), versionDir.getAbsolutePath());
Assert.assertTrue("Version directory should exist!", versionDir.exists());
// open store
@SuppressWarnings("unchecked") Serializer<Object> serializer = (Serializer<Object>) new DefaultSerializerFactory().getSerializer(serDef);
ReadOnlyStorageEngine engine = new ReadOnlyStorageEngine(storeName, searchStrategy, new RoutingStrategyFactory().updateRoutingStrategy(def, cluster), 0, storeDir, 1);
Store<Object, Object, Object> store = SerializingStore.wrap(engine, serializer, serializer, serializer);
// check values
for (Map.Entry<String, String> entry : values.entrySet()) {
String key = entry.getKey();
try {
List<Versioned<Object>> found = store.get(key, null);
Assert.assertEquals("Incorrect number of results", 1, found.size());
Assert.assertEquals(entry.getValue(), found.get(0).getValue());
} catch (VoldemortException e) {
throw new VoldemortException("Got an exception while trying to get key '" + key + "'.", e);
}
}
// also check the iterator - first key iterator...
try {
ClosableIterator<ByteArray> keyIterator = engine.keys();
if (!saveKeys) {
fail("Should have thrown an exception since this RO format does not support iterators");
}
int numElements = 0;
while (keyIterator.hasNext()) {
Assert.assertTrue(values.containsKey(serializer.toObject(keyIterator.next().get())));
numElements++;
}
Assert.assertEquals(numElements, values.size());
} catch (UnsupportedOperationException e) {
if (saveKeys) {
fail("Should not have thrown an exception since this RO format does support iterators");
}
}
// ... and entry iterator
try {
ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entryIterator = engine.entries();
if (!saveKeys) {
fail("Should have thrown an exception since this RO format does not support iterators");
}
int numElements = 0;
while (entryIterator.hasNext()) {
Pair<ByteArray, Versioned<byte[]>> entry = entryIterator.next();
Assert.assertEquals(values.get(serializer.toObject(entry.getFirst().get())), serializer.toObject(entry.getSecond().getValue()));
numElements++;
}
Assert.assertEquals(numElements, values.size());
} catch (UnsupportedOperationException e) {
if (saveKeys) {
fail("Should not have thrown an exception since this RO format does support iterators");
}
}
}
Aggregations