use of voldemort.common.OpTimeMap in project voldemort by voldemort.
the class VoldemortConfig method initializeStateFromProps.
/**
* This function populates the various strongly-typed variables of this class by
* extracting the values from {@link VoldemortConfig#allProps}.
*
* At this point, all defaults should have been resolved properly, so we can assume
* that all properties are present. If that's not the case, the correct behavior is
* to bubble up an UndefinedPropertyException.
*
* This code is isolated into its own function to prevent future code from trying to
* extract configurations from anywhere else besides {@link VoldemortConfig#allProps}.
*
* @throws UndefinedPropertyException if any required property has not been set.
*/
private void initializeStateFromProps() throws UndefinedPropertyException {
this.nodeId = this.allProps.getInt(NODE_ID, INVALID_NODE_ID);
this.voldemortHome = this.allProps.getString(VOLDEMORT_HOME);
this.dataDirectory = this.allProps.getString(DATA_DIRECTORY);
this.metadataDirectory = this.allProps.getString(METADATA_DIRECTORY);
this.bdbCacheSize = this.allProps.getBytes(BDB_CACHE_SIZE);
this.bdbWriteTransactions = this.allProps.getBoolean(BDB_WRITE_TRANSACTIONS);
this.bdbFlushTransactions = this.allProps.getBoolean(BDB_FLUSH_TRANSACTIONS);
this.bdbDataDirectory = this.allProps.getString(BDB_DATA_DIRECTORY);
this.bdbMaxLogFileSize = this.allProps.getBytes(BDB_MAX_LOGFILE_SIZE);
this.bdbBtreeFanout = this.allProps.getInt(BDB_BTREE_FANOUT);
this.bdbMaxDelta = this.allProps.getInt(BDB_MAX_DELTA);
this.bdbBinDelta = this.allProps.getInt(BDB_BIN_DELTA);
this.bdbCheckpointBytes = this.allProps.getLong(BDB_CHECKPOINT_INTERVAL_BYTES);
this.bdbCheckpointMs = this.allProps.getLong(BDB_CHECKPOINT_INTERVAL_MS);
this.bdbOneEnvPerStore = this.allProps.getBoolean(BDB_ONE_ENV_PER_STORE);
this.bdbCleanerMinFileUtilization = this.allProps.getInt(BDB_CLEANER_MIN_FILE_UTILIZATION);
this.bdbCleanerMinUtilization = this.allProps.getInt(BDB_CLEANER_MIN_UTILIZATION);
this.bdbCleanerThreads = this.allProps.getInt(BDB_CLEANER_THREADS);
this.bdbCleanerBytesInterval = this.allProps.getLong(BDB_CLEANER_INTERVAL_BYTES);
this.bdbCleanerLookAheadCacheSize = this.allProps.getInt(BDB_CLEANER_LOOKAHEAD_CACHE_SIZE);
this.bdbLockTimeoutMs = this.allProps.getLong(BDB_LOCK_TIMEOUT_MS);
this.bdbLockNLockTables = this.allProps.getInt(BDB_LOCK_N_LOCK_TABLES);
this.bdbLogFaultReadSize = this.allProps.getInt(BDB_LOG_FAULT_READ_SIZE);
this.bdbLogIteratorReadSize = this.allProps.getInt(BDB_LOG_ITERATOR_READ_SIZE);
this.bdbFairLatches = this.allProps.getBoolean(BDB_FAIR_LATCHES);
this.bdbCheckpointerHighPriority = this.allProps.getBoolean(BDB_CHECKPOINTER_HIGH_PRIORITY);
this.bdbCleanerMaxBatchFiles = this.allProps.getInt(BDB_CLEANER_MAX_BATCH_FILES);
this.bdbReadUncommitted = this.allProps.getBoolean(BDB_LOCK_READ_UNCOMMITTED);
this.bdbStatsCacheTtlMs = this.allProps.getLong(BDB_STATS_CACHE_TTL_MS);
this.bdbExposeSpaceUtilization = this.allProps.getBoolean(BDB_EXPOSE_SPACE_UTILIZATION);
this.bdbMinimumSharedCache = this.allProps.getLong(BDB_MINIMUM_SHARED_CACHE);
this.bdbCleanerLazyMigration = this.allProps.getBoolean(BDB_CLEANER_LAZY_MIGRATION);
this.bdbCacheModeEvictLN = this.allProps.getBoolean(BDB_CACHE_EVICTLN);
this.bdbMinimizeScanImpact = this.allProps.getBoolean(BDB_MINIMIZE_SCAN_IMPACT);
this.bdbPrefixKeysWithPartitionId = this.allProps.getBoolean(BDB_PREFIX_KEYS_WITH_PARTITIONID);
this.bdbLevelBasedEviction = this.allProps.getBoolean(BDB_EVICT_BY_LEVEL);
this.bdbCheckpointerOffForBatchWrites = this.allProps.getBoolean(BDB_CHECKPOINTER_OFF_BATCH_WRITES);
this.bdbCleanerFetchObsoleteSize = this.allProps.getBoolean(BDB_CLEANER_FETCH_OBSOLETE_SIZE);
this.bdbCleanerAdjustUtilization = this.allProps.getBoolean(BDB_CLEANER_ADJUST_UTILIZATION);
this.bdbRecoveryForceCheckpoint = this.allProps.getBoolean(BDB_RECOVERY_FORCE_CHECKPOINT);
this.bdbRawPropertyString = this.allProps.getString(BDB_RAW_PROPERTY_STRING);
this.numReadOnlyVersions = this.allProps.getInt(READONLY_BACKUPS);
this.readOnlySearchStrategy = this.allProps.getString(READONLY_SEARCH_STRATEGY);
this.readOnlyStorageDir = this.allProps.getString(READONLY_DATA_DIRECTORY);
this.readOnlyDeleteBackupTimeMs = this.allProps.getInt(READONLY_DELETE_BACKUP_MS);
this.readOnlyFetcherMaxBytesPerSecond = this.allProps.getBytes(FETCHER_MAX_BYTES_PER_SEC);
this.readOnlyFetcherReportingIntervalBytes = this.allProps.getBytes(FETCHER_REPORTING_INTERVAL_BYTES);
this.readOnlyFetcherThrottlerInterval = this.allProps.getInt(FETCHER_THROTTLER_INTERVAL);
this.readOnlyFetchRetryCount = this.allProps.getInt(FETCHER_RETRY_COUNT);
this.readOnlyFetchRetryDelayMs = this.allProps.getLong(FETCHER_RETRY_DELAY_MS);
this.readOnlyLoginIntervalMs = this.allProps.getLong(FETCHER_LOGIN_INTERVAL_MS);
this.defaultStorageSpaceQuotaInKB = this.allProps.getLong(DEFAULT_STORAGE_SPACE_QUOTA_IN_KB);
this.fetcherBufferSize = (int) this.allProps.getBytes(HDFS_FETCHER_BUFFER_SIZE);
this.fetcherSocketTimeout = this.allProps.getInt(HDFS_FETCHER_SOCKET_TIMEOUT);
this.readOnlyKeytabPath = this.allProps.getString(READONLY_KEYTAB_PATH);
this.readOnlyKerberosUser = this.allProps.getString(READONLY_KERBEROS_USER);
this.hadoopConfigPath = this.allProps.getString(READONLY_HADOOP_CONFIG_PATH);
this.readOnlyKerberosKdc = this.allProps.getString(READONLY_KERBEROS_KDC);
this.readOnlykerberosRealm = this.allProps.getString(READONLY_KERBEROS_REALM);
this.fileFetcherClass = this.allProps.getString(FILE_FETCHER_CLASS);
this.readOnlyStatsFileEnabled = this.allProps.getBoolean(READONLY_STATS_FILE_ENABLED);
this.readOnlyMaxVersionsStatsFile = this.allProps.getInt(READONLY_STATS_FILE_MAX_VERSIONS);
this.readOnlyMaxValueBufferAllocationSize = this.allProps.getInt(READONLY_MAX_VALUE_BUFFER_ALLOCATION_SIZE);
this.readOnlyCompressionCodec = this.allProps.getString(READONLY_COMPRESSION_CODEC);
this.readOnlyModifyProtocol = this.allProps.getString(READONLY_MODIFY_PROTOCOL);
this.readOnlyModifyPort = this.allProps.getInt(READONLY_MODIFY_PORT);
this.readOnlyOmitPort = this.allProps.getBoolean(READONLY_OMIT_PORT);
this.bouncyCastleEnabled = this.allProps.getBoolean(USE_BOUNCYCASTLE_FOR_SSL);
this.readOnlyBuildPrimaryReplicasOnly = this.allProps.getBoolean(READONLY_BUILD_PRIMARY_REPLICAS_ONLY);
this.highAvailabilityPushClusterId = this.allProps.getString(PUSH_HA_CLUSTER_ID);
this.highAvailabilityPushLockPath = this.allProps.getString(PUSH_HA_LOCK_PATH);
this.highAvailabilityPushLockImplementation = this.allProps.getString(PUSH_HA_LOCK_IMPLEMENTATION);
this.highAvailabilityPushMaxNodeFailures = this.allProps.getInt(PUSH_HA_MAX_NODE_FAILURES);
this.highAvailabilityPushEnabled = this.allProps.getBoolean(PUSH_HA_ENABLED);
this.highAvailabilityStateAutoCleanUp = this.allProps.getBoolean(PUSH_HA_STATE_AUTO_CLEANUP);
this.mysqlUsername = this.allProps.getString(MYSQL_USER);
this.mysqlPassword = this.allProps.getString(MYSQL_PASSWORD);
this.mysqlHost = this.allProps.getString(MYSQL_HOST);
this.mysqlPort = this.allProps.getInt(MYSQL_PORT);
this.mysqlDatabaseName = this.allProps.getString(MYSQL_DATABASE);
this.testingSlowQueueingDelays = new OpTimeMap(0);
this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_GET_MS));
this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_ALL_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_GETALL_MS));
this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_VERSION_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_GETVERSIONS_MS));
this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.PUT_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_PUT_MS));
this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.DELETE_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_DELETE_MS));
this.testingSlowConcurrentDelays = new OpTimeMap(0);
this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_GET_MS));
this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_ALL_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_GETALL_MS));
this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_VERSION_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_GETVERSIONS_MS));
this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.PUT_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_PUT_MS));
this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.DELETE_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_DELETE_MS));
this.maxThreads = this.allProps.getInt(MAX_THREADS);
this.coreThreads = this.allProps.getInt(CORE_THREADS);
// Admin client should have less threads but very high buffer size.
this.adminMaxThreads = this.allProps.getInt(ADMIN_MAX_THREADS);
this.adminCoreThreads = this.allProps.getInt(ADMIN_CORE_THREADS);
this.adminStreamBufferSize = (int) this.allProps.getBytes(ADMIN_STREAMS_BUFFER_SIZE);
this.adminConnectionTimeout = this.allProps.getInt(ADMIN_CLIENT_CONNECTION_TIMEOUT_SEC);
this.adminSocketTimeout = this.allProps.getInt(ADMIN_CLIENT_SOCKET_TIMEOUT_SEC);
this.streamMaxReadBytesPerSec = this.allProps.getBytes(STREAM_READ_BYTE_PER_SEC);
this.streamMaxWriteBytesPerSec = this.allProps.getBytes(STREAM_WRITE_BYTE_PER_SEC);
this.multiVersionStreamingPutsEnabled = this.allProps.getBoolean(USE_MULTI_VERSION_STREAMING_PUTS);
this.socketTimeoutMs = this.allProps.getInt(SOCKET_TIMEOUT_MS);
this.socketBufferSize = (int) this.allProps.getBytes(SOCKET_BUFFER_SIZE);
this.socketKeepAlive = this.allProps.getBoolean(SOCKET_KEEPALIVE);
this.useNioConnector = this.allProps.getBoolean(ENABLE_NIO_CONNECTOR);
this.nioConnectorKeepAlive = this.allProps.getBoolean(NIO_CONNECTOR_KEEPALIVE);
this.nioConnectorSelectors = this.allProps.getInt(NIO_CONNECTOR_SELECTORS);
this.nioAdminConnectorSelectors = this.allProps.getInt(NIO_ADMIN_CONNECTOR_SELECTORS);
this.nioAdminConnectorKeepAlive = this.allProps.getBoolean(NIO_ADMIN_CONNECTOR_KEEPALIVE);
// a value <= 0 forces the default to be used
this.nioAcceptorBacklog = this.allProps.getInt(NIO_ACCEPTOR_BACKLOG);
this.nioSelectorMaxHeartBeatTimeMs = this.allProps.getLong(NIO_SELECTOR_MAX_HEART_BEAT_TIME_MS);
this.clientSelectors = this.allProps.getInt(CLIENT_SELECTORS);
this.clientMaxConnectionsPerNode = this.allProps.getInt(CLIENT_MAX_CONNECTIONS_PER_NODE);
this.clientConnectionTimeoutMs = this.allProps.getInt(CLIENT_CONNECTION_TIMEOUT_MS);
this.clientRoutingTimeoutMs = this.allProps.getInt(CLIENT_ROUTING_TIMEOUT_MS);
this.clientTimeoutConfig = new TimeoutConfig(this.clientRoutingTimeoutMs, false);
this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_GET_TIMEOUT_MS));
this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_GETALL_TIMEOUT_MS));
this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_PUT_TIMEOUT_MS));
this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, this.allProps.getLong(CLIENT_ROUTING_GETVERSIONS_TIMEOUT_MS));
this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_DELETE_TIMEOUT_MS));
this.clientTimeoutConfig.setPartialGetAllAllowed(this.allProps.getBoolean(CLIENT_ROUTING_ALLOW_PARTIAL_GETALL));
this.clientMaxThreads = this.allProps.getInt(CLIENT_MAX_THREADS);
this.clientThreadIdleMs = this.allProps.getInt(CLIENT_THREAD_IDLE_MS);
this.clientMaxQueuedRequests = this.allProps.getInt(CLIENT_MAX_QUEUED_REQUESTS);
this.enableHttpServer = this.allProps.getBoolean(HTTP_ENABLE);
this.enableSocketServer = this.allProps.getBoolean(SOCKET_ENABLE);
this.enableAdminServer = this.allProps.getBoolean(ADMIN_ENABLE);
this.enableJmx = this.allProps.getBoolean(JMX_ENABLE);
this.enableSlop = this.allProps.getBoolean(SLOP_ENABLE);
this.enableSlopPusherJob = this.allProps.getBoolean(SLOP_PUSHER_ENABLE);
this.slopMaxWriteBytesPerSec = this.allProps.getBytes(SLOP_WRITE_BYTE_PER_SEC);
this.enableVerboseLogging = this.allProps.getBoolean(ENABLE_VERBOSE_LOGGING);
this.enableStatTracking = this.allProps.getBoolean(ENABLE_STAT_TRACKING);
this.enableServerRouting = this.allProps.getBoolean(ENABLE_SERVER_ROUTING);
this.enableMetadataChecking = this.allProps.getBoolean(ENABLE_METADATA_CHECKING);
this.enableGossip = this.allProps.getBoolean(ENABLE_GOSSIP);
this.enableRebalanceService = this.allProps.getBoolean(ENABLE_REBALANCING);
this.enableRepair = this.allProps.getBoolean(ENABLE_REPAIR);
this.enablePruneJob = this.allProps.getBoolean(ENABLE_PRUNEJOB);
this.enableSlopPurgeJob = this.allProps.getBoolean(ENABLE_SLOP_PURGE_JOB);
this.enableJmxClusterName = this.allProps.getBoolean(ENABLE_JMX_CLUSTERNAME);
this.enableQuotaLimiting = this.allProps.getBoolean(ENABLE_QUOTA_LIMITING);
this.gossipIntervalMs = this.allProps.getInt(GOSSIP_INTERVAL_MS);
this.slopMaxWriteBytesPerSec = this.allProps.getBytes(SLOP_WRITE_BYTE_PER_SEC1);
this.slopMaxReadBytesPerSec = this.allProps.getBytes(SLOP_READ_BYTE_PER_SEC);
this.slopStoreType = this.allProps.getString(SLOP_STORE_ENGINE);
this.slopFrequencyMs = this.allProps.getLong(SLOP_FREQUENCY_MS);
this.slopBatchSize = this.allProps.getInt(SLOP_BATCH_SIZE);
this.pusherType = this.allProps.getString(PUSHER_TYPE);
this.slopZonesDownToTerminate = this.allProps.getInt(SLOP_ZONES_TERMINATE);
this.autoPurgeDeadSlops = this.allProps.getBoolean(AUTO_PURGE_DEAD_SLOPS);
this.schedulerThreads = this.allProps.getInt(SCHEDULER_THREADS);
this.mayInterruptService = this.allProps.getBoolean(SERVICE_INTERRUPTIBLE);
this.numScanPermits = this.allProps.getInt(NUM_SCAN_PERMITS);
this.storageConfigurations = this.allProps.getList(STORAGE_CONFIGS);
this.retentionCleanupFirstStartTimeInHour = this.allProps.getInt(RETENTION_CLEANUP_FIRST_START_HOUR);
this.retentionCleanupFirstStartDayOfWeek = this.allProps.getInt(RETENTION_CLEANUP_FIRST_START_DAY);
this.retentionCleanupScheduledPeriodInHour = this.allProps.getInt(RETENTION_CLEANUP_PERIOD_HOURS);
this.retentionCleanupPinStartTime = this.allProps.getBoolean(RETENTION_CLEANUP_PIN_START_TIME);
this.enforceRetentionPolicyOnRead = this.allProps.getBoolean(ENFORCE_RETENTION_POLICY_ON_READ);
this.deleteExpiredValuesOnRead = this.allProps.getBoolean(DELETE_EXPIRED_VALUES_ON_READ);
this.requestFormatType = RequestFormatType.fromCode(this.allProps.getString(REQUEST_FORMAT));
// rebalancing parameters
this.rebalancingTimeoutSec = this.allProps.getLong(REBALANCING_TIMEOUT_SECONDS);
this.maxParallelStoresRebalancing = this.allProps.getInt(MAX_PARALLEL_STORES_REBALANCING);
this.usePartitionScanForRebalance = this.allProps.getBoolean(USE_PARTITION_SCAN_FOR_REBALANCE);
this.maxProxyPutThreads = this.allProps.getInt(MAX_PROXY_PUT_THREADS);
this.failureDetectorImplementation = this.allProps.getString(FAILUREDETECTOR_IMPLEMENTATION);
this.failureDetectorBannagePeriod = this.allProps.getLong(FAILUREDETECTOR_BANNAGE_PERIOD);
this.failureDetectorThreshold = this.allProps.getInt(FAILUREDETECTOR_THRESHOLD);
this.failureDetectorThresholdCountMinimum = this.allProps.getInt(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM);
this.failureDetectorThresholdInterval = this.allProps.getLong(FAILUREDETECTOR_THRESHOLD_INTERVAL);
this.failureDetectorAsyncRecoveryInterval = this.allProps.getLong(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL);
this.failureDetectorCatastrophicErrorTypes = this.allProps.getList(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES);
this.failureDetectorRequestLengthThreshold = this.allProps.getLong(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD);
// network class loader disable by default.
this.enableNetworkClassLoader = this.allProps.getBoolean(ENABLE_NETWORK_CLASSLOADER);
// TODO: REST-Server decide on the numbers
this.enableRestService = this.allProps.getBoolean(REST_ENABLE);
this.numRestServiceNettyServerBacklog = this.allProps.getInt(NUM_REST_SERVICE_NETTY_SERVER_BACKLOG);
this.numRestServiceNettyBossThreads = this.allProps.getInt(NUM_REST_SERVICE_NETTY_BOSS_THREADS);
this.numRestServiceNettyWorkerThreads = this.allProps.getInt(NUM_REST_SERVICE_NETTY_WORKER_THREADS);
this.numRestServiceStorageThreads = this.allProps.getInt(NUM_REST_SERVICE_STORAGE_THREADS);
this.restServiceStorageThreadPoolQueueSize = this.allProps.getInt(REST_SERVICE_STORAGE_THREAD_POOL_QUEUE_SIZE);
this.maxHttpAggregatedContentLength = this.allProps.getInt(MAX_HTTP_AGGREGATED_CONTENT_LENGTH);
this.repairJobMaxKeysScannedPerSec = this.allProps.getInt(REPAIRJOB_MAX_KEYS_SCANNED_PER_SEC);
this.pruneJobMaxKeysScannedPerSec = this.allProps.getInt(PRUNEJOB_MAX_KEYS_SCANNED_PER_SEC);
this.slopPurgeJobMaxKeysScannedPerSec = this.allProps.getInt(SLOP_PURGEJOB_MAX_KEYS_SCANNED_PER_SEC);
// RocksDB config
this.rocksdbDataDirectory = this.allProps.getString(ROCKSDB_DATA_DIR);
this.rocksdbPrefixKeysWithPartitionId = this.allProps.getBoolean(ROCKSDB_PREFIX_KEYS_WITH_PARTITIONID);
this.rocksdbEnableReadLocks = this.allProps.getBoolean(ROCKSDB_ENABLE_READ_LOCKS);
this.restrictedConfigs = this.allProps.getList(RESTRICTED_CONFIGS);
// Node Id auto detection configs
this.enableNodeIdDetection = this.allProps.getBoolean(ENABLE_NODE_ID_DETECTION, false);
// validation is defaulted based on node id detection.
this.validateNodeId = this.allProps.getBoolean(VALIDATE_NODE_ID);
this.nodeIdImplementation = new HostMatcher();
}
use of voldemort.common.OpTimeMap in project voldemort by voldemort.
the class SlowStorageEngineTest method setUp.
@Override
public void setUp() throws Exception {
super.setUp();
// Do not change the magic constants in the next two constructors! The
// unit tests assert on specific delays occurring.
OpTimeMap queued = new OpTimeMap(10, 20, 30, 40, 50);
OpTimeMap concurrent = new OpTimeMap(50, 40, 30, 20, 10);
this.store = new SlowStorageEngine<ByteArray, byte[], byte[]>(new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"), queued, concurrent);
}
Aggregations