Examples with Cluster voldemort.cluster.Cluster used on opensource projects

Example 6 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class RedirectingStore method redirectingPut.

/**
     * This is slightly different from other redirecting*** methods in that,
     * this updates the remote proxy node, with this put request, so we can
     * switch back to the old cluster topology if needed
     * 
     * @param key
     * @param value
     * @param transforms
     * @throws VoldemortException
     */
private void redirectingPut(ByteArray key, Versioned<byte[]> value, byte[] transforms) throws VoldemortException {
    Cluster currentCluster = metadata.getCluster();
    // TODO:refactor O(n) linear lookup of storedef here. Ideally should be
    // a hash lookup.
    StoreDefinition storeDef = metadata.getStoreDef(getName());
    /*
         * defensively, error out if this is a read-only store and someone is
         * doing puts against it. We don't to do extra work and fill the log
         * with errors in that case.
         */
    if (storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0) {
        throw new UnsupportedOperationException("put() not supported on read-only store");
    }
    BaseStoreRoutingPlan currentRoutingPlan = new BaseStoreRoutingPlan(currentCluster, storeDef);
    Integer redirectNode = getProxyNode(currentRoutingPlan, storeDef, key.get());
    /**
         * If I am rebalancing for this key, try to do remote get() if this node
         * does not have the key , put it locally first to get the correct
         * version ignoring any {@link ObsoleteVersionException}
         */
    if (redirectNode != null) {
        /*
             * first check if the key exists locally. If so, it means, it has
             * been moved over (either by a proxy fetch or background fetch) and
             * we are good simply issuing the put on top of that.
             */
        List<Versioned<byte[]>> vals = getInnerStore().get(key, transforms);
        if (vals.isEmpty()) {
            // if not, then go proxy fetch it
            if (logger.isTraceEnabled()) {
                logger.trace("Proxying GET (before PUT) on stealer:" + metadata.getNodeId() + " for  key " + ByteUtils.toHexString(key.get()) + " to node:" + redirectNode);
            }
            proxyGetAndLocalPut(key, redirectNode, transforms);
        }
    }
    // Here we are sure that the current node has caught up with the proxy
    // for this key. Moving on to the put logic.
    // put the data locally, if this step fails, there will be no proxy puts
    getInnerStore().put(key, value, transforms);
    // OVE). So do not send proxy puts in those cases.
    if (redirectNode != null && !currentRoutingPlan.getReplicationNodeList(key.get()).contains(redirectNode)) {
        AsyncProxyPutTask asyncProxyPutTask = new AsyncProxyPutTask(this, key, value, transforms, redirectNode);
        proxyPutStats.reportProxyPutSubmission();
        proxyPutWorkerPool.submit(asyncProxyPutTask);
    }
}

Example 7 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class MetadataStore method convertObjectToString.

/**
     * Converts Object to byte[] depending on the key
     * <p>
     * StoreRepository takes only StorageEngine<ByteArray,byte[]> and for
     * persistence on disk we need to convert them to String.<br>
     * 
     * @param key
     * @param value
     * @return
     */
@SuppressWarnings("unchecked")
private Versioned<String> convertObjectToString(String key, Versioned<Object> value) {
    String valueStr = "";
    if (CLUSTER_KEY.equals(key)) {
        valueStr = clusterMapper.writeCluster((Cluster) value.getValue());
    } else if (STORES_KEY.equals(key)) {
        valueStr = storeMapper.writeStoreList((List<StoreDefinition>) value.getValue());
    } else if (REBALANCING_STEAL_INFO.equals(key)) {
        RebalancerState rebalancerState = (RebalancerState) value.getValue();
        valueStr = rebalancerState.toJsonString();
    } else if (SERVER_STATE_KEY.equals(key) || NODE_ID_KEY.equals(key) || SLOP_STREAMING_ENABLED_KEY.equals(key) || PARTITION_STREAMING_ENABLED_KEY.equals(key) || READONLY_FETCH_ENABLED_KEY.equals(key) || QUOTA_ENFORCEMENT_ENABLED_KEY.equals(key)) {
        valueStr = value.getValue().toString();
    } else if (REBALANCING_SOURCE_CLUSTER_XML.equals(key)) {
        if (value.getValue() != null) {
            valueStr = clusterMapper.writeCluster((Cluster) value.getValue());
        }
    } else if (REBALANCING_SOURCE_STORES_XML.equals(key)) {
        if (value.getValue() != null) {
            valueStr = storeMapper.writeStoreList((List<StoreDefinition>) value.getValue());
        }
    } else if (this.storeNames.contains(key)) {
        valueStr = "<stores>";
        if (value.getValue() != null) {
            valueStr += value.getValue();
        }
        valueStr += "</stores>";
    } else {
        throw new VoldemortException("Unhandled key:'" + key + "' for Object to String serialization.");
    }
    return new Versioned<String>(valueStr, value.getVersion());
}

Example 8 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class MetadataStore method put.

/**
     * helper function to convert strings to bytes as needed.
     * 
     * @param key
     * @param value
     */
@SuppressWarnings("unchecked")
public void put(String key, Versioned<Object> value) {
    // acquire write lock
    writeLock.lock();
    try {
        if (this.storeNames.contains(key) || key.equals(STORES_KEY)) {
            // Check for backwards compatibility
            List<StoreDefinition> storeDefinitions = (List<StoreDefinition>) value.getValue();
            StoreDefinitionUtils.validateSchemasAsNeeded(storeDefinitions);
            // If the put is on the entire stores.xml key, delete the
            // additional stores which do not exist in the specified
            // stores.xml
            Set<String> storeNamesToDelete = new HashSet<String>();
            for (String storeName : this.storeNames) {
                storeNamesToDelete.add(storeName);
            }
            // Add / update the list of store definitions specified in the
            // value
            StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
            // Update the STORES directory and the corresponding entry in
            // metadata cache
            Set<String> specifiedStoreNames = new HashSet<String>();
            for (StoreDefinition storeDef : storeDefinitions) {
                specifiedStoreNames.add(storeDef.getName());
                String storeDefStr = mapper.writeStore(storeDef);
                Versioned<String> versionedValueStr = new Versioned<String>(storeDefStr, value.getVersion());
                this.storeDefinitionsStorageEngine.put(storeDef.getName(), versionedValueStr, "");
                // Update the metadata cache
                this.metadataCache.put(storeDef.getName(), new Versioned<Object>(storeDefStr, value.getVersion()));
            }
            if (key.equals(STORES_KEY)) {
                storeNamesToDelete.removeAll(specifiedStoreNames);
                resetStoreDefinitions(storeNamesToDelete);
            }
            // Re-initialize the store definitions
            initStoreDefinitions(value.getVersion());
            // Update routing strategies
            updateRoutingStrategies(getCluster(), getStoreDefList());
        } else if (METADATA_KEYS.contains(key)) {
            // try inserting into inner store first
            putInner(key, convertObjectToString(key, value));
            // cache all keys if innerStore put succeeded
            metadataCache.put(key, value);
            // do special stuff if needed
            if (CLUSTER_KEY.equals(key)) {
                updateRoutingStrategies((Cluster) value.getValue(), getStoreDefList());
            } else if (NODE_ID_KEY.equals(key)) {
                initNodeId(getNodeIdNoLock());
            } else if (SYSTEM_STORES_KEY.equals(key))
                throw new VoldemortException("Cannot overwrite system store definitions");
        } else {
            throw new VoldemortException("Unhandled Key:" + key + " for MetadataStore put()");
        }
    } finally {
        writeLock.unlock();
    }
}

Example 9 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class RequestFileFilter method main.

/**
     * Filter requests specified in a file, generating a new file containing
     * only requests destined for a specific node.
     * 
     * @param args See usage for more information
     * @throws Exception In case of I/O or Voldemort-specific errors
     */
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("node", "[REQUIRED] node id").withRequiredArg().ofType(Integer.class).describedAs("node id");
    parser.accepts("store-name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url");
    parser.accepts("input", "[REQUIRED] input request file").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] output file").withRequiredArg().describedAs("output-file");
    parser.accepts("string-keys");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "node", "store-name", "url", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    int nodeId = (Integer) options.valueOf("node");
    String storeName = (String) options.valueOf("store-name");
    String bootstrapURL = (String) options.valueOf("url");
    String inputFile = (String) options.valueOf("input");
    String outputFile = (String) options.valueOf("output");
    boolean stringKeys = options.has("string-keys");
    AdminClient adminClient = new AdminClient(bootstrapURL);
    List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId).getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : storeDefinitionList) {
        if (storeName.equals(def.getName())) {
            storeDefinition = def;
        }
    }
    if (storeDefinition == null) {
        Utils.croak("No store found with name\"" + storeName + "\"");
    }
    Cluster cluster = adminClient.metadataMgmtOps.getRemoteCluster(nodeId).getValue();
    Node node = null;
    try {
        node = cluster.getNodeById(nodeId);
    } catch (VoldemortException e) {
        Utils.croak("Can't find a node with id " + nodeId);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    try {
        new RequestFileFilter(storeDefinition, routingStrategy, inputFile, outputFile, node, stringKeys).filter();
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}

Example 10 with Cluster

use of voldemort.cluster.Cluster in project voldemort by voldemort.

the class RoutedStoreParallelismTest method main.

public static void main(String[] args) throws Throwable {
    OptionParser parser = new OptionParser();
    parser.accepts("num-keys", "The number of keys to submit for retrieval  Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\"  Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\"  Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-nodes", "The number of nodes  Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-clients", "The number of threads to make requests concurrently  Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
    parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\"  Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
    parser.accepts("help", "This help");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printUsage(System.out, parser);
    }
    final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
    int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
    int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
    int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
    int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
    int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
    int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
    System.err.println("num-keys : " + numKeys);
    System.err.println("max-connections : " + maxConnectionsPerNode);
    System.err.println("max-threads : " + maxThreads);
    System.err.println("num-nodes : " + numNodes);
    System.err.println("num-slow-nodes : " + numSlowNodes);
    System.err.println("delay : " + delay);
    System.err.println("num-clients : " + numClients);
    ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
    String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
    StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
    VoldemortServer[] servers = new VoldemortServer[numNodes];
    int[][] partitionMap = new int[numNodes][1];
    for (int i = 0; i < numNodes; i++) {
        partitionMap[i][0] = i;
    }
    Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
    Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
    for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
        serverMap.put(i, servers[i]);
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
        if (i < numSlowNodes)
            store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
        StoreRepository storeRepository = servers[i].getStoreRepository();
        storeRepository.addLocalStore(store);
    }
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
        stores.put(node.getId(), socketStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
    FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
    final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
    ExecutorService runner = Executors.newFixedThreadPool(numClients);
    long start = System.nanoTime();
    try {
        for (int i = 0; i < numClients; i++) {
            runner.submit(new Runnable() {

                public void run() {
                    for (int i = 0; i < numKeys; i++) {
                        ByteArray key = new ByteArray(("test-key-" + i).getBytes());
                        try {
                            routedStore.get(key, null);
                        } catch (VoldemortException e) {
                        //
                        }
                    }
                }
            });
        }
        runner.shutdown();
        runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        long time = (System.nanoTime() - start) / Time.NS_PER_MS;
        System.err.println("Time: " + time + " ms.");
    } finally {
        runner.shutdown();
    }
    if (failureDetector != null)
        failureDetector.destroy();
    for (VoldemortServer server : serverMap.values()) server.stop();
    if (routedStoreThreadPool != null)
        routedStoreThreadPool.shutdown();
    System.exit(0);
}