Examples with TimeoutConfig voldemort.client.TimeoutConfig used on opensource projects

Example 1 with TimeoutConfig

use of voldemort.client.TimeoutConfig in project voldemort by voldemort.

the class RESTClientConfig method setProperties.

/**
     * Set the values using the specified Properties object.
     * 
     * @param properties Properties object containing specific property values
     *        for the RESTClient config
     * 
     *        Note: We're using the same property names as that in ClientConfig
     *        for backwards compatibility.
     */
private void setProperties(Properties properties) {
    Props props = new Props(properties);
    if (props.containsKey(ClientConfig.ENABLE_JMX_PROPERTY)) {
        this.setEnableJmx(props.getBoolean(ClientConfig.ENABLE_JMX_PROPERTY));
    }
    if (props.containsKey(ClientConfig.BOOTSTRAP_URLS_PROPERTY)) {
        List<String> urls = props.getList(ClientConfig.BOOTSTRAP_URLS_PROPERTY);
        if (urls.size() > 0) {
            setHttpBootstrapURL(urls.get(0));
        }
    }
    if (props.containsKey(ClientConfig.MAX_TOTAL_CONNECTIONS_PROPERTY)) {
        setMaxR2ConnectionPoolSize(props.getInt(ClientConfig.MAX_TOTAL_CONNECTIONS_PROPERTY, maxR2ConnectionPoolSize));
    }
    if (props.containsKey(ClientConfig.ROUTING_TIMEOUT_MS_PROPERTY))
        this.setTimeoutMs(props.getLong(ClientConfig.ROUTING_TIMEOUT_MS_PROPERTY, timeoutMs), TimeUnit.MILLISECONDS);
    // By default, make all the timeouts equal to routing timeout
    timeoutConfig = new TimeoutConfig(timeoutMs, false);
    if (props.containsKey(ClientConfig.GETALL_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, props.getInt(ClientConfig.GETALL_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.GET_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE, props.getInt(ClientConfig.GET_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.PUT_ROUTING_TIMEOUT_MS_PROPERTY)) {
        long putTimeoutMs = props.getInt(ClientConfig.PUT_ROUTING_TIMEOUT_MS_PROPERTY);
        timeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, putTimeoutMs);
        // By default, use the same thing for getVersions() also
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, putTimeoutMs);
    }
    // of course, if someone overrides it, we will respect that
    if (props.containsKey(ClientConfig.GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, props.getInt(ClientConfig.GET_VERSIONS_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.DELETE_ROUTING_TIMEOUT_MS_PROPERTY))
        timeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE, props.getInt(ClientConfig.DELETE_ROUTING_TIMEOUT_MS_PROPERTY));
    if (props.containsKey(ClientConfig.ALLOW_PARTIAL_GETALLS_PROPERTY))
        timeoutConfig.setPartialGetAllAllowed(props.getBoolean(ClientConfig.ALLOW_PARTIAL_GETALLS_PROPERTY));
}

Example 2 with TimeoutConfig

use of voldemort.client.TimeoutConfig in project voldemort by voldemort.

the class ClientConnectionStressTest method main.

@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.accepts(CONNECTION_TIMEOUT, "Connection timeout (ms)").withRequiredArg().ofType(Integer.class);
    parser.accepts(ROUTING_TIMEOUT, "Routing timeout (ms)").withRequiredArg().ofType(Integer.class);
    parser.accepts(SOCKET_TIMEOUT, "Socket timeout (ms)").withRequiredArg().ofType(Integer.class);
    parser.accepts(MAX_CONNECTIONS, "Max connections per node").withRequiredArg().ofType(Integer.class);
    parser.accepts(MAX_CONNECTIONS_TOTAL, "Max total connections").withRequiredArg().ofType(Integer.class);
    parser.accepts(MAX_THREADS, "Max threads").withRequiredArg().ofType(Integer.class);
    parser.accepts(SELECTORS, "Number of NIO selectors").withRequiredArg().ofType(Integer.class);
    parser.accepts(SOCKET_BUFFER_SIZE, "Socket buffer size").withRequiredArg().ofType(Integer.class);
    parser.accepts(REQS, "Requests per session").withRequiredArg().ofType(Integer.class);
    parser.accepts(CONNECTIONS, "Total connections to make").withRequiredArg().ofType(Integer.class);
    parser.accepts("help");
    OptionSet options = parser.parse(args);
    List<String> rest = (List<String>) options.nonOptionArguments();
    if (rest.size() < 2 || options.has("help")) {
        parser.printHelpOn(System.err);
        System.err.println("Usage: ClientConnectionStressTest <options> url store-name");
        System.exit(0);
    }
    String url = rest.get(0);
    String storeName = rest.get(1);
    Integer connsTotal = CmdUtils.valueOf(options, CONNECTIONS, 100);
    Integer reqs = CmdUtils.valueOf(options, REQS, 1000);
    ClientConfig config = new ClientConfig();
    if (options.has(CONNECTION_TIMEOUT))
        config.setConnectionTimeout((Integer) options.valueOf(CONNECTION_TIMEOUT), TimeUnit.MILLISECONDS);
    if (options.has(ROUTING_TIMEOUT))
        config.setTimeoutConfig(new TimeoutConfig(TimeUnit.MILLISECONDS.toMillis((Integer) options.valueOf(ROUTING_TIMEOUT)), false));
    if (options.has(SOCKET_TIMEOUT))
        config.setSocketTimeout((Integer) options.valueOf(SOCKET_TIMEOUT), TimeUnit.MILLISECONDS);
    if (options.has(MAX_CONNECTIONS))
        config.setMaxConnectionsPerNode((Integer) options.valueOf(MAX_CONNECTIONS));
    if (options.has(MAX_THREADS))
        config.setMaxThreads((Integer) options.valueOf(MAX_THREADS));
    if (options.has(SELECTORS))
        config.setSelectors((Integer) options.valueOf(SELECTORS));
    if (options.has(SOCKET_BUFFER_SIZE))
        config.setSocketBufferSize((Integer) options.valueOf(SOCKET_BUFFER_SIZE));
    config.setBootstrapUrls(url);
    ClientConnectionStressTest test = new ClientConnectionStressTest(config, storeName, config.getMaxThreads(), connsTotal, reqs);
    test.execute();
}

Example 3 with TimeoutConfig

use of voldemort.client.TimeoutConfig in project voldemort by voldemort.

the class VoldemortConfig method initializeStateFromProps.

/**
     * This function populates the various strongly-typed variables of this class by
     * extracting the values from {@link VoldemortConfig#allProps}.
     *
     * At this point, all defaults should have been resolved properly, so we can assume
     * that all properties are present. If that's not the case, the correct behavior is
     * to bubble up an UndefinedPropertyException.
     *
     * This code is isolated into its own function to prevent future code from trying to
     * extract configurations from anywhere else besides {@link VoldemortConfig#allProps}.
     *
     * @throws UndefinedPropertyException if any required property has not been set.
     */
private void initializeStateFromProps() throws UndefinedPropertyException {
    this.nodeId = this.allProps.getInt(NODE_ID, INVALID_NODE_ID);
    this.voldemortHome = this.allProps.getString(VOLDEMORT_HOME);
    this.dataDirectory = this.allProps.getString(DATA_DIRECTORY);
    this.metadataDirectory = this.allProps.getString(METADATA_DIRECTORY);
    this.bdbCacheSize = this.allProps.getBytes(BDB_CACHE_SIZE);
    this.bdbWriteTransactions = this.allProps.getBoolean(BDB_WRITE_TRANSACTIONS);
    this.bdbFlushTransactions = this.allProps.getBoolean(BDB_FLUSH_TRANSACTIONS);
    this.bdbDataDirectory = this.allProps.getString(BDB_DATA_DIRECTORY);
    this.bdbMaxLogFileSize = this.allProps.getBytes(BDB_MAX_LOGFILE_SIZE);
    this.bdbBtreeFanout = this.allProps.getInt(BDB_BTREE_FANOUT);
    this.bdbMaxDelta = this.allProps.getInt(BDB_MAX_DELTA);
    this.bdbBinDelta = this.allProps.getInt(BDB_BIN_DELTA);
    this.bdbCheckpointBytes = this.allProps.getLong(BDB_CHECKPOINT_INTERVAL_BYTES);
    this.bdbCheckpointMs = this.allProps.getLong(BDB_CHECKPOINT_INTERVAL_MS);
    this.bdbOneEnvPerStore = this.allProps.getBoolean(BDB_ONE_ENV_PER_STORE);
    this.bdbCleanerMinFileUtilization = this.allProps.getInt(BDB_CLEANER_MIN_FILE_UTILIZATION);
    this.bdbCleanerMinUtilization = this.allProps.getInt(BDB_CLEANER_MIN_UTILIZATION);
    this.bdbCleanerThreads = this.allProps.getInt(BDB_CLEANER_THREADS);
    this.bdbCleanerBytesInterval = this.allProps.getLong(BDB_CLEANER_INTERVAL_BYTES);
    this.bdbCleanerLookAheadCacheSize = this.allProps.getInt(BDB_CLEANER_LOOKAHEAD_CACHE_SIZE);
    this.bdbLockTimeoutMs = this.allProps.getLong(BDB_LOCK_TIMEOUT_MS);
    this.bdbLockNLockTables = this.allProps.getInt(BDB_LOCK_N_LOCK_TABLES);
    this.bdbLogFaultReadSize = this.allProps.getInt(BDB_LOG_FAULT_READ_SIZE);
    this.bdbLogIteratorReadSize = this.allProps.getInt(BDB_LOG_ITERATOR_READ_SIZE);
    this.bdbFairLatches = this.allProps.getBoolean(BDB_FAIR_LATCHES);
    this.bdbCheckpointerHighPriority = this.allProps.getBoolean(BDB_CHECKPOINTER_HIGH_PRIORITY);
    this.bdbCleanerMaxBatchFiles = this.allProps.getInt(BDB_CLEANER_MAX_BATCH_FILES);
    this.bdbReadUncommitted = this.allProps.getBoolean(BDB_LOCK_READ_UNCOMMITTED);
    this.bdbStatsCacheTtlMs = this.allProps.getLong(BDB_STATS_CACHE_TTL_MS);
    this.bdbExposeSpaceUtilization = this.allProps.getBoolean(BDB_EXPOSE_SPACE_UTILIZATION);
    this.bdbMinimumSharedCache = this.allProps.getLong(BDB_MINIMUM_SHARED_CACHE);
    this.bdbCleanerLazyMigration = this.allProps.getBoolean(BDB_CLEANER_LAZY_MIGRATION);
    this.bdbCacheModeEvictLN = this.allProps.getBoolean(BDB_CACHE_EVICTLN);
    this.bdbMinimizeScanImpact = this.allProps.getBoolean(BDB_MINIMIZE_SCAN_IMPACT);
    this.bdbPrefixKeysWithPartitionId = this.allProps.getBoolean(BDB_PREFIX_KEYS_WITH_PARTITIONID);
    this.bdbLevelBasedEviction = this.allProps.getBoolean(BDB_EVICT_BY_LEVEL);
    this.bdbCheckpointerOffForBatchWrites = this.allProps.getBoolean(BDB_CHECKPOINTER_OFF_BATCH_WRITES);
    this.bdbCleanerFetchObsoleteSize = this.allProps.getBoolean(BDB_CLEANER_FETCH_OBSOLETE_SIZE);
    this.bdbCleanerAdjustUtilization = this.allProps.getBoolean(BDB_CLEANER_ADJUST_UTILIZATION);
    this.bdbRecoveryForceCheckpoint = this.allProps.getBoolean(BDB_RECOVERY_FORCE_CHECKPOINT);
    this.bdbRawPropertyString = this.allProps.getString(BDB_RAW_PROPERTY_STRING);
    this.numReadOnlyVersions = this.allProps.getInt(READONLY_BACKUPS);
    this.readOnlySearchStrategy = this.allProps.getString(READONLY_SEARCH_STRATEGY);
    this.readOnlyStorageDir = this.allProps.getString(READONLY_DATA_DIRECTORY);
    this.readOnlyDeleteBackupTimeMs = this.allProps.getInt(READONLY_DELETE_BACKUP_MS);
    this.readOnlyFetcherMaxBytesPerSecond = this.allProps.getBytes(FETCHER_MAX_BYTES_PER_SEC);
    this.readOnlyFetcherReportingIntervalBytes = this.allProps.getBytes(FETCHER_REPORTING_INTERVAL_BYTES);
    this.readOnlyFetcherThrottlerInterval = this.allProps.getInt(FETCHER_THROTTLER_INTERVAL);
    this.readOnlyFetchRetryCount = this.allProps.getInt(FETCHER_RETRY_COUNT);
    this.readOnlyFetchRetryDelayMs = this.allProps.getLong(FETCHER_RETRY_DELAY_MS);
    this.readOnlyLoginIntervalMs = this.allProps.getLong(FETCHER_LOGIN_INTERVAL_MS);
    this.defaultStorageSpaceQuotaInKB = this.allProps.getLong(DEFAULT_STORAGE_SPACE_QUOTA_IN_KB);
    this.fetcherBufferSize = (int) this.allProps.getBytes(HDFS_FETCHER_BUFFER_SIZE);
    this.fetcherSocketTimeout = this.allProps.getInt(HDFS_FETCHER_SOCKET_TIMEOUT);
    this.readOnlyKeytabPath = this.allProps.getString(READONLY_KEYTAB_PATH);
    this.readOnlyKerberosUser = this.allProps.getString(READONLY_KERBEROS_USER);
    this.hadoopConfigPath = this.allProps.getString(READONLY_HADOOP_CONFIG_PATH);
    this.readOnlyKerberosKdc = this.allProps.getString(READONLY_KERBEROS_KDC);
    this.readOnlykerberosRealm = this.allProps.getString(READONLY_KERBEROS_REALM);
    this.fileFetcherClass = this.allProps.getString(FILE_FETCHER_CLASS);
    this.readOnlyStatsFileEnabled = this.allProps.getBoolean(READONLY_STATS_FILE_ENABLED);
    this.readOnlyMaxVersionsStatsFile = this.allProps.getInt(READONLY_STATS_FILE_MAX_VERSIONS);
    this.readOnlyMaxValueBufferAllocationSize = this.allProps.getInt(READONLY_MAX_VALUE_BUFFER_ALLOCATION_SIZE);
    this.readOnlyCompressionCodec = this.allProps.getString(READONLY_COMPRESSION_CODEC);
    this.readOnlyModifyProtocol = this.allProps.getString(READONLY_MODIFY_PROTOCOL);
    this.readOnlyModifyPort = this.allProps.getInt(READONLY_MODIFY_PORT);
    this.readOnlyOmitPort = this.allProps.getBoolean(READONLY_OMIT_PORT);
    this.bouncyCastleEnabled = this.allProps.getBoolean(USE_BOUNCYCASTLE_FOR_SSL);
    this.readOnlyBuildPrimaryReplicasOnly = this.allProps.getBoolean(READONLY_BUILD_PRIMARY_REPLICAS_ONLY);
    this.highAvailabilityPushClusterId = this.allProps.getString(PUSH_HA_CLUSTER_ID);
    this.highAvailabilityPushLockPath = this.allProps.getString(PUSH_HA_LOCK_PATH);
    this.highAvailabilityPushLockImplementation = this.allProps.getString(PUSH_HA_LOCK_IMPLEMENTATION);
    this.highAvailabilityPushMaxNodeFailures = this.allProps.getInt(PUSH_HA_MAX_NODE_FAILURES);
    this.highAvailabilityPushEnabled = this.allProps.getBoolean(PUSH_HA_ENABLED);
    this.highAvailabilityStateAutoCleanUp = this.allProps.getBoolean(PUSH_HA_STATE_AUTO_CLEANUP);
    this.mysqlUsername = this.allProps.getString(MYSQL_USER);
    this.mysqlPassword = this.allProps.getString(MYSQL_PASSWORD);
    this.mysqlHost = this.allProps.getString(MYSQL_HOST);
    this.mysqlPort = this.allProps.getInt(MYSQL_PORT);
    this.mysqlDatabaseName = this.allProps.getString(MYSQL_DATABASE);
    this.testingSlowQueueingDelays = new OpTimeMap(0);
    this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_GET_MS));
    this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_ALL_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_GETALL_MS));
    this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.GET_VERSION_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_GETVERSIONS_MS));
    this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.PUT_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_PUT_MS));
    this.testingSlowQueueingDelays.setOpTime(VoldemortOpCode.DELETE_OP_CODE, this.allProps.getInt(TESTING_SLOW_QUEUEING_DELETE_MS));
    this.testingSlowConcurrentDelays = new OpTimeMap(0);
    this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_GET_MS));
    this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_ALL_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_GETALL_MS));
    this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.GET_VERSION_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_GETVERSIONS_MS));
    this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.PUT_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_PUT_MS));
    this.testingSlowConcurrentDelays.setOpTime(VoldemortOpCode.DELETE_OP_CODE, this.allProps.getInt(TESTING_SLOW_CONCURRENT_DELETE_MS));
    this.maxThreads = this.allProps.getInt(MAX_THREADS);
    this.coreThreads = this.allProps.getInt(CORE_THREADS);
    // Admin client should have less threads but very high buffer size.
    this.adminMaxThreads = this.allProps.getInt(ADMIN_MAX_THREADS);
    this.adminCoreThreads = this.allProps.getInt(ADMIN_CORE_THREADS);
    this.adminStreamBufferSize = (int) this.allProps.getBytes(ADMIN_STREAMS_BUFFER_SIZE);
    this.adminConnectionTimeout = this.allProps.getInt(ADMIN_CLIENT_CONNECTION_TIMEOUT_SEC);
    this.adminSocketTimeout = this.allProps.getInt(ADMIN_CLIENT_SOCKET_TIMEOUT_SEC);
    this.streamMaxReadBytesPerSec = this.allProps.getBytes(STREAM_READ_BYTE_PER_SEC);
    this.streamMaxWriteBytesPerSec = this.allProps.getBytes(STREAM_WRITE_BYTE_PER_SEC);
    this.multiVersionStreamingPutsEnabled = this.allProps.getBoolean(USE_MULTI_VERSION_STREAMING_PUTS);
    this.socketTimeoutMs = this.allProps.getInt(SOCKET_TIMEOUT_MS);
    this.socketBufferSize = (int) this.allProps.getBytes(SOCKET_BUFFER_SIZE);
    this.socketKeepAlive = this.allProps.getBoolean(SOCKET_KEEPALIVE);
    this.useNioConnector = this.allProps.getBoolean(ENABLE_NIO_CONNECTOR);
    this.nioConnectorKeepAlive = this.allProps.getBoolean(NIO_CONNECTOR_KEEPALIVE);
    this.nioConnectorSelectors = this.allProps.getInt(NIO_CONNECTOR_SELECTORS);
    this.nioAdminConnectorSelectors = this.allProps.getInt(NIO_ADMIN_CONNECTOR_SELECTORS);
    this.nioAdminConnectorKeepAlive = this.allProps.getBoolean(NIO_ADMIN_CONNECTOR_KEEPALIVE);
    // a value <= 0 forces the default to be used
    this.nioAcceptorBacklog = this.allProps.getInt(NIO_ACCEPTOR_BACKLOG);
    this.nioSelectorMaxHeartBeatTimeMs = this.allProps.getLong(NIO_SELECTOR_MAX_HEART_BEAT_TIME_MS);
    this.clientSelectors = this.allProps.getInt(CLIENT_SELECTORS);
    this.clientMaxConnectionsPerNode = this.allProps.getInt(CLIENT_MAX_CONNECTIONS_PER_NODE);
    this.clientConnectionTimeoutMs = this.allProps.getInt(CLIENT_CONNECTION_TIMEOUT_MS);
    this.clientRoutingTimeoutMs = this.allProps.getInt(CLIENT_ROUTING_TIMEOUT_MS);
    this.clientTimeoutConfig = new TimeoutConfig(this.clientRoutingTimeoutMs, false);
    this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.GET_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_GET_TIMEOUT_MS));
    this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_GETALL_TIMEOUT_MS));
    this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.PUT_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_PUT_TIMEOUT_MS));
    this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.GET_VERSION_OP_CODE, this.allProps.getLong(CLIENT_ROUTING_GETVERSIONS_TIMEOUT_MS));
    this.clientTimeoutConfig.setOperationTimeout(VoldemortOpCode.DELETE_OP_CODE, this.allProps.getInt(CLIENT_ROUTING_DELETE_TIMEOUT_MS));
    this.clientTimeoutConfig.setPartialGetAllAllowed(this.allProps.getBoolean(CLIENT_ROUTING_ALLOW_PARTIAL_GETALL));
    this.clientMaxThreads = this.allProps.getInt(CLIENT_MAX_THREADS);
    this.clientThreadIdleMs = this.allProps.getInt(CLIENT_THREAD_IDLE_MS);
    this.clientMaxQueuedRequests = this.allProps.getInt(CLIENT_MAX_QUEUED_REQUESTS);
    this.enableHttpServer = this.allProps.getBoolean(HTTP_ENABLE);
    this.enableSocketServer = this.allProps.getBoolean(SOCKET_ENABLE);
    this.enableAdminServer = this.allProps.getBoolean(ADMIN_ENABLE);
    this.enableJmx = this.allProps.getBoolean(JMX_ENABLE);
    this.enableSlop = this.allProps.getBoolean(SLOP_ENABLE);
    this.enableSlopPusherJob = this.allProps.getBoolean(SLOP_PUSHER_ENABLE);
    this.slopMaxWriteBytesPerSec = this.allProps.getBytes(SLOP_WRITE_BYTE_PER_SEC);
    this.enableVerboseLogging = this.allProps.getBoolean(ENABLE_VERBOSE_LOGGING);
    this.enableStatTracking = this.allProps.getBoolean(ENABLE_STAT_TRACKING);
    this.enableServerRouting = this.allProps.getBoolean(ENABLE_SERVER_ROUTING);
    this.enableMetadataChecking = this.allProps.getBoolean(ENABLE_METADATA_CHECKING);
    this.enableGossip = this.allProps.getBoolean(ENABLE_GOSSIP);
    this.enableRebalanceService = this.allProps.getBoolean(ENABLE_REBALANCING);
    this.enableRepair = this.allProps.getBoolean(ENABLE_REPAIR);
    this.enablePruneJob = this.allProps.getBoolean(ENABLE_PRUNEJOB);
    this.enableSlopPurgeJob = this.allProps.getBoolean(ENABLE_SLOP_PURGE_JOB);
    this.enableJmxClusterName = this.allProps.getBoolean(ENABLE_JMX_CLUSTERNAME);
    this.enableQuotaLimiting = this.allProps.getBoolean(ENABLE_QUOTA_LIMITING);
    this.gossipIntervalMs = this.allProps.getInt(GOSSIP_INTERVAL_MS);
    this.slopMaxWriteBytesPerSec = this.allProps.getBytes(SLOP_WRITE_BYTE_PER_SEC1);
    this.slopMaxReadBytesPerSec = this.allProps.getBytes(SLOP_READ_BYTE_PER_SEC);
    this.slopStoreType = this.allProps.getString(SLOP_STORE_ENGINE);
    this.slopFrequencyMs = this.allProps.getLong(SLOP_FREQUENCY_MS);
    this.slopBatchSize = this.allProps.getInt(SLOP_BATCH_SIZE);
    this.pusherType = this.allProps.getString(PUSHER_TYPE);
    this.slopZonesDownToTerminate = this.allProps.getInt(SLOP_ZONES_TERMINATE);
    this.autoPurgeDeadSlops = this.allProps.getBoolean(AUTO_PURGE_DEAD_SLOPS);
    this.schedulerThreads = this.allProps.getInt(SCHEDULER_THREADS);
    this.mayInterruptService = this.allProps.getBoolean(SERVICE_INTERRUPTIBLE);
    this.numScanPermits = this.allProps.getInt(NUM_SCAN_PERMITS);
    this.storageConfigurations = this.allProps.getList(STORAGE_CONFIGS);
    this.retentionCleanupFirstStartTimeInHour = this.allProps.getInt(RETENTION_CLEANUP_FIRST_START_HOUR);
    this.retentionCleanupFirstStartDayOfWeek = this.allProps.getInt(RETENTION_CLEANUP_FIRST_START_DAY);
    this.retentionCleanupScheduledPeriodInHour = this.allProps.getInt(RETENTION_CLEANUP_PERIOD_HOURS);
    this.retentionCleanupPinStartTime = this.allProps.getBoolean(RETENTION_CLEANUP_PIN_START_TIME);
    this.enforceRetentionPolicyOnRead = this.allProps.getBoolean(ENFORCE_RETENTION_POLICY_ON_READ);
    this.deleteExpiredValuesOnRead = this.allProps.getBoolean(DELETE_EXPIRED_VALUES_ON_READ);
    this.requestFormatType = RequestFormatType.fromCode(this.allProps.getString(REQUEST_FORMAT));
    // rebalancing parameters
    this.rebalancingTimeoutSec = this.allProps.getLong(REBALANCING_TIMEOUT_SECONDS);
    this.maxParallelStoresRebalancing = this.allProps.getInt(MAX_PARALLEL_STORES_REBALANCING);
    this.usePartitionScanForRebalance = this.allProps.getBoolean(USE_PARTITION_SCAN_FOR_REBALANCE);
    this.maxProxyPutThreads = this.allProps.getInt(MAX_PROXY_PUT_THREADS);
    this.failureDetectorImplementation = this.allProps.getString(FAILUREDETECTOR_IMPLEMENTATION);
    this.failureDetectorBannagePeriod = this.allProps.getLong(FAILUREDETECTOR_BANNAGE_PERIOD);
    this.failureDetectorThreshold = this.allProps.getInt(FAILUREDETECTOR_THRESHOLD);
    this.failureDetectorThresholdCountMinimum = this.allProps.getInt(FAILUREDETECTOR_THRESHOLD_COUNTMINIMUM);
    this.failureDetectorThresholdInterval = this.allProps.getLong(FAILUREDETECTOR_THRESHOLD_INTERVAL);
    this.failureDetectorAsyncRecoveryInterval = this.allProps.getLong(FAILUREDETECTOR_ASYNCRECOVERY_INTERVAL);
    this.failureDetectorCatastrophicErrorTypes = this.allProps.getList(FAILUREDETECTOR_CATASTROPHIC_ERROR_TYPES);
    this.failureDetectorRequestLengthThreshold = this.allProps.getLong(FAILUREDETECTOR_REQUEST_LENGTH_THRESHOLD);
    // network class loader disable by default.
    this.enableNetworkClassLoader = this.allProps.getBoolean(ENABLE_NETWORK_CLASSLOADER);
    // TODO: REST-Server decide on the numbers
    this.enableRestService = this.allProps.getBoolean(REST_ENABLE);
    this.numRestServiceNettyServerBacklog = this.allProps.getInt(NUM_REST_SERVICE_NETTY_SERVER_BACKLOG);
    this.numRestServiceNettyBossThreads = this.allProps.getInt(NUM_REST_SERVICE_NETTY_BOSS_THREADS);
    this.numRestServiceNettyWorkerThreads = this.allProps.getInt(NUM_REST_SERVICE_NETTY_WORKER_THREADS);
    this.numRestServiceStorageThreads = this.allProps.getInt(NUM_REST_SERVICE_STORAGE_THREADS);
    this.restServiceStorageThreadPoolQueueSize = this.allProps.getInt(REST_SERVICE_STORAGE_THREAD_POOL_QUEUE_SIZE);
    this.maxHttpAggregatedContentLength = this.allProps.getInt(MAX_HTTP_AGGREGATED_CONTENT_LENGTH);
    this.repairJobMaxKeysScannedPerSec = this.allProps.getInt(REPAIRJOB_MAX_KEYS_SCANNED_PER_SEC);
    this.pruneJobMaxKeysScannedPerSec = this.allProps.getInt(PRUNEJOB_MAX_KEYS_SCANNED_PER_SEC);
    this.slopPurgeJobMaxKeysScannedPerSec = this.allProps.getInt(SLOP_PURGEJOB_MAX_KEYS_SCANNED_PER_SEC);
    // RocksDB config
    this.rocksdbDataDirectory = this.allProps.getString(ROCKSDB_DATA_DIR);
    this.rocksdbPrefixKeysWithPartitionId = this.allProps.getBoolean(ROCKSDB_PREFIX_KEYS_WITH_PARTITIONID);
    this.rocksdbEnableReadLocks = this.allProps.getBoolean(ROCKSDB_ENABLE_READ_LOCKS);
    this.restrictedConfigs = this.allProps.getList(RESTRICTED_CONFIGS);
    // Node Id auto detection configs
    this.enableNodeIdDetection = this.allProps.getBoolean(ENABLE_NODE_ID_DETECTION, false);
    // validation is defaulted based on node id detection.
    this.validateNodeId = this.allProps.getBoolean(VALIDATE_NODE_ID);
    this.nodeIdImplementation = new HostMatcher();
}

Example 4 with TimeoutConfig

use of voldemort.client.TimeoutConfig in project voldemort by voldemort.

the class RoutedStoreTest method testPartialGetAllZZZ.

/**
     * Tests that getAll returns partial results in a 3 zone cluster (with a
     * node down).
     */
@Test
public void testPartialGetAllZZZ() throws Exception {
    // Set replication factors for a 3 zone cluster
    HashMap<Integer, Integer> zoneReplicationFactor = Maps.newHashMap();
    zoneReplicationFactor.put(0, 1);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    // Create a store with RF=3, Required reads = 3 and zone count reads = 2
    // This ensures that a GET operation requires a response from all 3
    // nodes (from the respective 3 zones)
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).setReplicationFactor(3).setPreferredReads(3).setRequiredReads(3).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(2).setZoneCountWrites(1).setZoneReplicationFactor(zoneReplicationFactor).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    // create nodes with varying speeds - 100ms, 200ms, 300ms
    for (int i = 0; i < 3; i++) {
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(100 * (i + 1), new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        // Create zoned nodes - one in each zone (0, 1, 2)
        nodes.add(new Node(i, "none", 0, 0, 0, i, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    TimeoutConfig timeoutConfig = new TimeoutConfig(1500, true);
    // This means, the getall will only succeed on two of the nodes
    timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, 250);
    RoutedStoreFactory routedStoreFactory = createFactory();
    List<Zone> zones = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        LinkedList<Integer> zoneProximityList = Lists.newLinkedList();
        Set<Integer> zoneIds = Sets.newHashSet(0, 1, 2);
        zoneIds.remove(i);
        zoneProximityList.addAll(zoneIds);
        zones.add(new Zone(i, zoneProximityList));
    }
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes, zones), definition, stores, failureDetector, createConfig(timeoutConfig));
    /* do some puts so we have some data to test getalls */
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        routedStore.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    /* 1. positive test; if partial is on, should get something back */
    Map<ByteArray, List<Versioned<byte[]>>> all = routedStore.getAll(expectedValues.keySet(), null);
    assert (expectedValues.size() > all.size());
    /* 2. negative test; if partial is off, should fail the whole operation */
    timeoutConfig.setPartialGetAllAllowed(false);
    try {
        all = routedStore.getAll(expectedValues.keySet(), null);
        fail("Should have failed");
    } catch (Exception e) {
    // Expected
    }
}

Example 5 with TimeoutConfig

use of voldemort.client.TimeoutConfig in project voldemort by voldemort.

the class RoutedStoreTest method testPartialGetAll.

/**
     * Tests that getAll returns partial results
     */
@Test
public void testPartialGetAll() throws Exception {
    // create a store with rf=1 i.e disjoint partitions
    StoreDefinition definition = new StoreDefinitionBuilder().setName("test").setType("foo").setKeySerializer(new SerializerDefinition("test")).setValueSerializer(new SerializerDefinition("test")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    List<Node> nodes = new ArrayList<Node>();
    // create nodes with varying speeds - 100ms, 200ms, 300ms
    for (int i = 0; i < 3; i++) {
        Store<ByteArray, byte[], byte[]> store = new SleepyStore<ByteArray, byte[], byte[]>(100 * (i + 1), new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test"));
        stores.put(i, store);
        List<Integer> partitions = Arrays.asList(i);
        nodes.add(new Node(i, "none", 0, 0, 0, partitions));
    }
    setFailureDetector(stores);
    routedStoreThreadPool = Executors.newFixedThreadPool(3);
    TimeoutConfig timeoutConfig = new TimeoutConfig(1500, true);
    // This means, the getall will only succeed on two of the nodes
    timeoutConfig.setOperationTimeout(VoldemortOpCode.GET_ALL_OP_CODE, 250);
    RoutedStoreFactory routedStoreFactory = createFactory();
    RoutedStore routedStore = routedStoreFactory.create(new Cluster("test", nodes), definition, stores, failureDetector, createConfig(timeoutConfig));
    /* do some puts so we have some data to test getalls */
    Map<ByteArray, byte[]> expectedValues = Maps.newHashMap();
    for (byte i = 1; i < 11; ++i) {
        ByteArray key = new ByteArray(new byte[] { i });
        byte[] value = new byte[] { (byte) (i + 50) };
        routedStore.put(key, Versioned.value(value), null);
        expectedValues.put(key, value);
    }
    /* 1. positive test; if partial is on, should get something back */
    Map<ByteArray, List<Versioned<byte[]>>> all = routedStore.getAll(expectedValues.keySet(), null);
    assert (expectedValues.size() > all.size());
    /* 2. negative test; if partial is off, should fail the whole operation */
    timeoutConfig.setPartialGetAllAllowed(false);
    try {
        all = routedStore.getAll(expectedValues.keySet(), null);
        fail("Should have failed");
    } catch (Exception e) {
    }
}