Search in sources :

Example 16 with ClusterMapper

use of voldemort.xml.ClusterMapper in project voldemort by voldemort.

the class VoldemortAdminTool method main.

@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    // This is a generic argument that should be eventually supported by all
    // RW operations.
    // If you omit this argument the operation will be executed in a "batch"
    // mode which is useful for scripting
    // Otherwise you will be presented with a summary of changes and with a
    // Y/N prompt
    parser.accepts("auto", "[OPTIONAL] enable auto/batch mode");
    parser.accepts("help", "print help information");
    parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url").ofType(String.class);
    parser.accepts("node", "node id").withRequiredArg().describedAs("node-id").ofType(Integer.class);
    parser.accepts("delete-partitions", "Delete partitions").withRequiredArg().describedAs("partition-ids").withValuesSeparatedBy(',').ofType(Integer.class);
    parser.accepts("restore", "Restore from replication [ Optional parallelism param - Default - 5 ]").withOptionalArg().describedAs("parallelism").ofType(Integer.class);
    parser.accepts("ascii", "Fetch keys as ASCII");
    parser.accepts("fetch-keys", "Fetch keys").withOptionalArg().describedAs("partition-ids").withValuesSeparatedBy(',').ofType(Integer.class);
    parser.accepts("fetch-entries", "Fetch full entries").withOptionalArg().describedAs("partition-ids").withValuesSeparatedBy(',').ofType(Integer.class);
    parser.accepts("outdir", "Output directory").withRequiredArg().describedAs("output-directory").ofType(String.class);
    parser.accepts("nodes", "list of nodes").withRequiredArg().describedAs("nodes").withValuesSeparatedBy(',').ofType(Integer.class);
    parser.accepts("stores", "Store names").withRequiredArg().describedAs("store-names").withValuesSeparatedBy(',').ofType(String.class);
    parser.accepts("store", "Store name for querying keys").withRequiredArg().describedAs("store-name").ofType(String.class);
    parser.accepts("add-stores", "Add stores in this stores.xml").withRequiredArg().describedAs("stores.xml containing just the new stores").ofType(String.class);
    parser.accepts("delete-store", "Delete store").withRequiredArg().describedAs("store-name").ofType(String.class);
    parser.accepts("update-entries", "Insert or update entries").withRequiredArg().describedAs("input-directory").ofType(String.class);
    parser.accepts("get-metadata", "retreive metadata information " + MetadataStore.METADATA_KEYS).withOptionalArg().describedAs("metadata-key").ofType(String.class);
    parser.accepts("check-metadata", "retreive metadata information from all nodes and checks if they are consistent across [ " + MetadataStore.CLUSTER_KEY + " | " + MetadataStore.STORES_KEY + " | " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + " | " + MetadataStore.SERVER_STATE_KEY + " ]").withRequiredArg().describedAs("metadata-key").ofType(String.class);
    parser.accepts("ro-metadata", "retrieve version information [current | max | storage-format]").withRequiredArg().describedAs("type").ofType(String.class);
    parser.accepts("truncate", "truncate a store").withRequiredArg().describedAs("store-name").ofType(String.class);
    parser.accepts("set-metadata", "Forceful setting of metadata [ " + MetadataStore.CLUSTER_KEY + " | " + MetadataStore.STORES_KEY + " | " + MetadataStore.SERVER_STATE_KEY + " | " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + " | " + MetadataStore.REBALANCING_STEAL_INFO + " ]").withRequiredArg().describedAs("metadata-key").ofType(String.class);
    parser.accepts("set-metadata-value", "The value for the set-metadata [ " + MetadataStore.CLUSTER_KEY + " | " + MetadataStore.STORES_KEY + ", " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + ", " + MetadataStore.REBALANCING_STEAL_INFO + " ] - xml file location, [ " + MetadataStore.SERVER_STATE_KEY + " ] - " + MetadataStore.VoldemortState.NORMAL_SERVER + "," + MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER + "," + MetadataStore.VoldemortState.OFFLINE_SERVER).withRequiredArg().describedAs("metadata-value").ofType(String.class);
    parser.accepts("update-store-defs", "Update the [" + MetadataStore.STORES_KEY + "] with the new value for only the specified stores in update-value.");
    parser.accepts("update-store-value", "The value for update-store-defs ] - xml file location").withRequiredArg().describedAs("stores-xml-value").ofType(String.class);
    parser.accepts("set-metadata-pair", "Atomic setting of metadata pair [ " + MetadataStore.CLUSTER_KEY + " & " + MetadataStore.STORES_KEY + " ]").withRequiredArg().describedAs("metadata-keys-pair").withValuesSeparatedBy(',').ofType(String.class);
    parser.accepts("set-metadata-value-pair", "The value for the set-metadata pair [ " + MetadataStore.CLUSTER_KEY + " & " + MetadataStore.STORES_KEY + " ]").withRequiredArg().describedAs("metadata-value-pair").withValuesSeparatedBy(',').ofType(String.class);
    parser.accepts("clear-rebalancing-metadata", "Remove the metadata related to rebalancing");
    parser.accepts("async", "a) Get a list of async job ids [get] b) Stop async job ids [stop] ").withRequiredArg().describedAs("op-type").ofType(String.class);
    parser.accepts("async-id", "Comma separated list of async ids to stop").withOptionalArg().describedAs("job-ids").withValuesSeparatedBy(',').ofType(Integer.class);
    parser.accepts("repair-job", "Clean after rebalancing is done");
    parser.accepts("prune-job", "Prune versioned put data, after rebalancing");
    parser.accepts("purge-slops", "Purge the slop stores selectively, based on nodeId or zoneId");
    parser.accepts("native-backup", "Perform a native backup").withRequiredArg().describedAs("store-name").ofType(String.class);
    parser.accepts("backup-dir").withRequiredArg().describedAs("backup-directory").ofType(String.class);
    parser.accepts("backup-timeout").withRequiredArg().describedAs("minutes to wait for backup completion, default 30 mins").ofType(Integer.class);
    parser.accepts("backup-verify", "If provided, backup will also verify checksum (with extra overhead)");
    parser.accepts("backup-incremental", "Perform an incremental backup for point-in-time recovery." + " By default backup has latest consistent snapshot.");
    parser.accepts("zone", "zone id").withRequiredArg().describedAs("zone-id").ofType(Integer.class);
    parser.accepts("rollback", "rollback a store").withRequiredArg().describedAs("store-name").ofType(String.class);
    parser.accepts("version", "Push version of store to rollback to").withRequiredArg().describedAs("version").ofType(Long.class);
    parser.accepts("verify-metadata-version", "Verify the version of Metadata on all the cluster nodes");
    parser.accepts("synchronize-metadata-version", "Synchronize the metadata versions across all the nodes.");
    parser.accepts("reserve-memory", "Memory in MB to reserve for the store").withRequiredArg().describedAs("size-in-mb").ofType(Long.class);
    parser.accepts("query-key", "Get values of a key on specific node").withRequiredArg().describedAs("query-key").ofType(String.class);
    parser.accepts("query-key-format", "Format of the query key. Can be one of [hex|readable]").withRequiredArg().describedAs("key-format").ofType(String.class);
    parser.accepts("show-routing-plan", "Routing plan of the specified keys").withRequiredArg().describedAs("keys-to-be-routed").withValuesSeparatedBy(',').ofType(String.class);
    parser.accepts("mirror-from-url", "Cluster url to mirror data from").withRequiredArg().describedAs("mirror-cluster-bootstrap-url").ofType(String.class);
    parser.accepts("mirror-node", "Node id in the mirror cluster to mirror from").withRequiredArg().describedAs("id-of-mirror-node").ofType(Integer.class);
    parser.accepts("fetch-orphaned", "Fetch any orphaned keys/entries in the node");
    parser.accepts("set-quota", "Enforce some quota on the servers").withRequiredArg().describedAs("quota-type").ofType(String.class);
    parser.accepts("quota-value", "Value of the quota enforced on the servers").withRequiredArg().describedAs("quota-value").ofType(String.class);
    parser.accepts("unset-quota", "Remove some quota already enforced on the servers").withRequiredArg().describedAs("quota-type").ofType(String.class);
    // TODO add a way to retrieve all quotas for a given store.
    parser.accepts("get-quota", "Retrieve some quota already enforced on the servers").withRequiredArg().describedAs("quota-type").ofType(String.class);
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printHelp(System.out, parser);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "url", "node");
    if (missing.size() > 0) {
        // options; all these can live without explicit node ids
        if (!(missing.equals(ImmutableSet.of("node")) && (options.has("add-stores") || options.has("delete-store") || options.has("ro-metadata") || options.has("set-metadata") || options.has("update-store-defs") || options.has("set-metadata-pair") || options.has("get-metadata") || options.has("check-metadata")) || options.has("truncate") || options.has("clear-rebalancing-metadata") || options.has("async") || options.has("native-backup") || options.has("rollback") || options.has("verify-metadata-version") || options.has("reserve-memory") || options.has("purge-slops") || options.has("show-routing-plan") || options.has("query-key") || options.has("set-quota") || options.has("unset-quota") || options.has("get-quota") || options.has("synchronize-metadata-version"))) {
            System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
            printHelp(System.err, parser);
            System.exit(1);
        }
    }
    try {
        String url = (String) options.valueOf("url");
        Integer nodeId = CmdUtils.valueOf(options, "node", -1);
        int parallelism = CmdUtils.valueOf(options, "restore", 5);
        Integer zoneId = CmdUtils.valueOf(options, "zone", -1);
        AdminClient adminClient = new AdminClient(url);
        List<String> storeNames = null;
        if (options.has("store") && options.has("stores")) {
            throw new VoldemortException("Must not specify both --stores and --store options");
        } else if (options.has("stores")) {
            storeNames = (List<String>) options.valuesOf("stores");
        } else if (options.has("store")) {
            storeNames = Arrays.asList((String) options.valueOf("store"));
        }
        String outputDir = null;
        if (options.has("outdir")) {
            outputDir = (String) options.valueOf("outdir");
        }
        if (options.has("add-stores")) {
            String storesXml = (String) options.valueOf("add-stores");
            executeAddStores(adminClient, storesXml, nodeId);
        } else if (options.has("async")) {
            String asyncKey = (String) options.valueOf("async");
            List<Integer> asyncIds = null;
            if (options.hasArgument("async-id"))
                asyncIds = (List<Integer>) options.valuesOf("async-id");
            executeAsync(nodeId, adminClient, asyncKey, asyncIds);
        } else if (options.has("check-metadata")) {
            String metadataKey = (String) options.valueOf("check-metadata");
            executeCheckMetadata(adminClient, metadataKey);
        } else if (options.has("delete-partitions")) {
            System.out.println("Starting delete-partitions");
            List<Integer> partitionIdList = (List<Integer>) options.valuesOf("delete-partitions");
            executeDeletePartitions(nodeId, adminClient, partitionIdList, storeNames);
            System.out.println("Finished delete-partitions");
        } else if (options.has("ro-metadata")) {
            String type = (String) options.valueOf("ro-metadata");
            executeROMetadata(nodeId, adminClient, storeNames, type);
        } else if (options.has("reserve-memory")) {
            if (!options.has("stores")) {
                Utils.croak("Specify the list of stores to reserve memory");
            }
            long reserveMB = (Long) options.valueOf("reserve-memory");
            adminClient.quotaMgmtOps.reserveMemory(nodeId, storeNames, reserveMB);
        } else if (options.has("get-metadata")) {
            String metadataKey = ALL_METADATA;
            if (options.hasArgument("get-metadata")) {
                metadataKey = (String) options.valueOf("get-metadata");
            }
            executeGetMetadata(nodeId, adminClient, metadataKey, outputDir);
        } else if (options.has("mirror-from-url")) {
            if (!options.has("mirror-node")) {
                Utils.croak("Specify the mirror node to fetch from");
            }
            if (nodeId == -1) {
                System.err.println("Cannot run mirroring without node id");
                System.exit(1);
            }
            Integer mirrorNodeId = CmdUtils.valueOf(options, "mirror-node", -1);
            if (mirrorNodeId == -1) {
                System.err.println("Cannot run mirroring without mirror node id");
                System.exit(1);
            }
            adminClient.restoreOps.mirrorData(nodeId, mirrorNodeId, (String) options.valueOf("mirror-from-url"), storeNames);
        } else if (options.has("clear-rebalancing-metadata")) {
            executeClearRebalancing(nodeId, adminClient);
        } else if (options.has("prune-job")) {
            if (storeNames == null) {
                Utils.croak("Must specify --stores to run the prune job");
            }
            executePruneJob(nodeId, adminClient, storeNames);
        } else if (options.has("fetch-keys")) {
            boolean useAscii = options.has("ascii");
            System.out.println("Starting fetch keys");
            List<Integer> partitionIdList = null;
            if (options.hasArgument("fetch-keys"))
                partitionIdList = (List<Integer>) options.valuesOf("fetch-keys");
            executeFetchKeys(nodeId, adminClient, partitionIdList, outputDir, storeNames, useAscii, options.has("fetch-orphaned"));
        } else if (options.has("repair-job")) {
            executeRepairJob(nodeId, adminClient);
        } else if (options.has("set-metadata-pair")) {
            List<String> metadataKeyPair = (List<String>) options.valuesOf("set-metadata-pair");
            if (metadataKeyPair.size() != 2) {
                throw new VoldemortException("Missing set-metadata-pair keys (only two keys are needed and allowed)");
            }
            if (!options.has("set-metadata-value-pair")) {
                throw new VoldemortException("Missing set-metadata-value-pair");
            } else {
                List<String> metadataValuePair = (List<String>) options.valuesOf("set-metadata-value-pair");
                if (metadataValuePair.size() != 2) {
                    throw new VoldemortException("Missing set-metadata--value-pair values (only two values are needed and allowed)");
                }
                if (metadataKeyPair.contains(MetadataStore.CLUSTER_KEY) && metadataKeyPair.contains(MetadataStore.STORES_KEY)) {
                    ClusterMapper clusterMapper = new ClusterMapper();
                    StoreDefinitionsMapper storeDefsMapper = new StoreDefinitionsMapper();
                    // original metadata
                    Integer nodeIdToGetStoreXMLFrom = nodeId;
                    if (nodeId < 0) {
                        Collection<Node> nodes = adminClient.getAdminClientCluster().getNodes();
                        if (nodes.isEmpty()) {
                            throw new VoldemortException("No nodes in this cluster");
                        } else {
                            nodeIdToGetStoreXMLFrom = nodes.iterator().next().getId();
                        }
                    }
                    Versioned<String> storesXML = adminClient.metadataMgmtOps.getRemoteMetadata(nodeIdToGetStoreXMLFrom, MetadataStore.STORES_KEY);
                    List<StoreDefinition> oldStoreDefs = storeDefsMapper.readStoreList(new StringReader(storesXML.getValue()));
                    String clusterXMLPath = metadataValuePair.get(metadataKeyPair.indexOf(MetadataStore.CLUSTER_KEY));
                    clusterXMLPath = clusterXMLPath.replace("~", System.getProperty("user.home"));
                    if (!Utils.isReadableFile(clusterXMLPath))
                        throw new VoldemortException("Cluster xml file path incorrect");
                    Cluster cluster = clusterMapper.readCluster(new File(clusterXMLPath));
                    String storesXMLPath = metadataValuePair.get(metadataKeyPair.indexOf(MetadataStore.STORES_KEY));
                    storesXMLPath = storesXMLPath.replace("~", System.getProperty("user.home"));
                    if (!Utils.isReadableFile(storesXMLPath))
                        throw new VoldemortException("Stores definition xml file path incorrect");
                    List<StoreDefinition> newStoreDefs = storeDefsMapper.readStoreList(new File(storesXMLPath));
                    StoreDefinitionUtils.validateSchemasAsNeeded(newStoreDefs);
                    executeSetMetadataPair(nodeId, adminClient, MetadataStore.CLUSTER_KEY, clusterMapper.writeCluster(cluster), MetadataStore.STORES_KEY, storeDefsMapper.writeStoreList(newStoreDefs));
                    executeUpdateMetadataVersionsOnStores(adminClient, oldStoreDefs, newStoreDefs);
                } else {
                    throw new VoldemortException("set-metadata-pair keys should be <cluster.xml, stores.xml>");
                }
            }
        } else if (options.has("set-metadata")) {
            String metadataKey = (String) options.valueOf("set-metadata");
            if (!options.has("set-metadata-value")) {
                throw new VoldemortException("Missing set-metadata-value");
            } else {
                String metadataValue = (String) options.valueOf("set-metadata-value");
                if (metadataKey.compareTo(MetadataStore.CLUSTER_KEY) == 0 || metadataKey.compareTo(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML) == 0) {
                    if (!Utils.isReadableFile(metadataValue))
                        throw new VoldemortException("Cluster xml file path incorrect");
                    ClusterMapper mapper = new ClusterMapper();
                    Cluster newCluster = mapper.readCluster(new File(metadataValue));
                    if (options.has("auto")) {
                        executeSetMetadata(nodeId, adminClient, metadataKey, mapper.writeCluster(newCluster));
                    } else {
                        if (confirmMetadataUpdate(nodeId, adminClient, mapper.writeCluster(newCluster))) {
                            executeSetMetadata(nodeId, adminClient, metadataKey, mapper.writeCluster(newCluster));
                        } else {
                            System.out.println("New metadata has not been set");
                        }
                    }
                } else if (metadataKey.compareTo(MetadataStore.SERVER_STATE_KEY) == 0) {
                    VoldemortState newState = VoldemortState.valueOf(metadataValue);
                    executeSetMetadata(nodeId, adminClient, MetadataStore.SERVER_STATE_KEY, newState.toString());
                } else if (metadataKey.compareTo(MetadataStore.STORES_KEY) == 0) {
                    if (!Utils.isReadableFile(metadataValue))
                        throw new VoldemortException("Stores definition xml file path incorrect");
                    StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
                    List<StoreDefinition> newStoreDefs = mapper.readStoreList(new File(metadataValue));
                    StoreDefinitionUtils.validateSchemasAsNeeded(newStoreDefs);
                    // original metadata
                    Integer nodeIdToGetStoreXMLFrom = nodeId;
                    if (nodeId < 0) {
                        Collection<Node> nodes = adminClient.getAdminClientCluster().getNodes();
                        if (nodes.isEmpty()) {
                            throw new VoldemortException("No nodes in this cluster");
                        } else {
                            nodeIdToGetStoreXMLFrom = nodes.iterator().next().getId();
                        }
                    }
                    Versioned<String> storesXML = adminClient.metadataMgmtOps.getRemoteMetadata(nodeIdToGetStoreXMLFrom, MetadataStore.STORES_KEY);
                    List<StoreDefinition> oldStoreDefs = mapper.readStoreList(new StringReader(storesXML.getValue()));
                    if (options.has("auto")) {
                        executeSetMetadata(nodeId, adminClient, MetadataStore.STORES_KEY, mapper.writeStoreList(newStoreDefs));
                        executeUpdateMetadataVersionsOnStores(adminClient, oldStoreDefs, newStoreDefs);
                    } else {
                        if (confirmMetadataUpdate(nodeId, adminClient, storesXML.getValue())) {
                            executeSetMetadata(nodeId, adminClient, MetadataStore.STORES_KEY, mapper.writeStoreList(newStoreDefs));
                            if (nodeId >= 0) {
                                System.err.println("WARNING: Metadata version update of stores goes to all servers, " + "although this set-metadata oprations only goes to node " + nodeId);
                            }
                            executeUpdateMetadataVersionsOnStores(adminClient, oldStoreDefs, newStoreDefs);
                        } else {
                            System.out.println("New metadata has not been set");
                        }
                    }
                } else if (metadataKey.compareTo(MetadataStore.REBALANCING_STEAL_INFO) == 0) {
                    if (!Utils.isReadableFile(metadataValue))
                        throw new VoldemortException("Rebalancing steal info file path incorrect");
                    String rebalancingStealInfoJsonString = FileUtils.readFileToString(new File(metadataValue));
                    RebalancerState state = RebalancerState.create(rebalancingStealInfoJsonString);
                    executeSetMetadata(nodeId, adminClient, MetadataStore.REBALANCING_STEAL_INFO, state.toJsonString());
                } else {
                    throw new VoldemortException("Incorrect metadata key");
                }
            }
        } else if (options.has("update-store-defs")) {
            if (!options.has("update-store-value")) {
                throw new VoldemortException("Missing update-store-value for update-store-defs");
            } else {
                String storesXmlValue = (String) options.valueOf("update-store-value");
                if (!Utils.isReadableFile(storesXmlValue))
                    throw new VoldemortException("Stores definition xml file path incorrect");
                StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
                List<StoreDefinition> newStoreDefs = mapper.readStoreList(new File(storesXmlValue));
                StoreDefinitionUtils.validateSchemasAsNeeded(newStoreDefs);
                if (options.has("auto")) {
                    executeUpdateStoreDefinitions(nodeId, adminClient, newStoreDefs);
                } else {
                    if (confirmMetadataUpdate(nodeId, adminClient, newStoreDefs)) {
                        executeUpdateStoreDefinitions(nodeId, adminClient, newStoreDefs);
                        if (nodeId >= 0) {
                            System.err.println("WARNING: Metadata version update of stores goes to all servers, " + "although this set-metadata oprations only goes to node " + nodeId);
                        }
                    } else {
                        System.out.println("New metadata has not been set");
                    }
                }
                System.out.println("The store definitions have been successfully updated.");
            }
        } else if (options.has("native-backup")) {
            if (!options.has("backup-dir")) {
                Utils.croak("A backup directory must be specified with backup-dir option");
            }
            String backupDir = (String) options.valueOf("backup-dir");
            String storeName = (String) options.valueOf("native-backup");
            int timeout = CmdUtils.valueOf(options, "backup-timeout", 30);
            adminClient.storeMntOps.nativeBackup(nodeId, storeName, backupDir, timeout, options.has("backup-verify"), options.has("backup-incremental"));
        } else if (options.has("rollback")) {
            if (!options.has("version")) {
                Utils.croak("A read-only push version must be specified with rollback option");
            }
            String storeName = (String) options.valueOf("rollback");
            long pushVersion = (Long) options.valueOf("version");
            executeRollback(nodeId, storeName, pushVersion, adminClient);
        } else if (options.has("query-key")) {
            String key = (String) options.valueOf("query-key");
            String keyFormat = (String) options.valueOf("query-key-format");
            if (keyFormat == null) {
                keyFormat = "hex";
            }
            if (!keyFormat.equals("hex") && !keyFormat.equals("readable")) {
                throw new VoldemortException("--query-key-format must be hex or readable");
            }
            executeQueryKey(nodeId, adminClient, storeNames, key, keyFormat);
        } else if (options.has("restore")) {
            if (nodeId == -1) {
                System.err.println("Cannot run restore without node id");
                System.exit(1);
            }
            System.out.println("Starting restore");
            adminClient.restoreOps.restoreDataFromReplications(nodeId, parallelism, zoneId);
            System.out.println("Finished restore");
        } else if (options.has("delete-store")) {
            String storeName = (String) options.valueOf("delete-store");
            executeDeleteStore(adminClient, storeName, nodeId);
        } else if (options.has("truncate")) {
            String storeName = (String) options.valueOf("truncate");
            executeTruncateStore(nodeId, adminClient, storeName);
        } else if (options.has("update-entries")) {
            String inputDir = (String) options.valueOf("update-entries");
            executeUpdateEntries(nodeId, adminClient, storeNames, inputDir);
        } else if (options.has("fetch-entries")) {
            boolean useAscii = options.has("ascii");
            System.out.println("Starting fetch entries");
            List<Integer> partitionIdList = null;
            if (options.hasArgument("fetch-entries"))
                partitionIdList = (List<Integer>) options.valuesOf("fetch-entries");
            executeFetchEntries(nodeId, adminClient, partitionIdList, outputDir, storeNames, useAscii, options.has("fetch-orphaned"));
        } else if (options.has("purge-slops")) {
            List<Integer> nodesToPurge = null;
            if (options.has("nodes")) {
                nodesToPurge = (List<Integer>) options.valuesOf("nodes");
            }
            if (nodesToPurge == null && zoneId == -1 && storeNames == null) {
                Utils.croak("Must specify atleast one of --nodes, --zone-id or --stores with --purge-slops");
            }
            executePurgeSlops(adminClient, nodesToPurge, zoneId, storeNames);
        } else if (options.has("synchronize-metadata-version")) {
            synchronizeMetadataVersion(adminClient);
        } else if (options.has("verify-metadata-version")) {
            checkMetadataVersion(adminClient);
        } else if (options.has("show-routing-plan")) {
            if (!options.has("store")) {
                Utils.croak("Must specify the store the keys belong to using --store ");
            }
            String storeName = (String) options.valueOf("store");
            List<String> keysToRoute = (List<String>) options.valuesOf("show-routing-plan");
            if (keysToRoute == null || keysToRoute.size() == 0) {
                Utils.croak("Must specify comma separated keys list in hex format");
            }
            executeShowRoutingPlan(adminClient, storeName, keysToRoute);
        } else if (options.has("set-quota")) {
            String quotaType = (String) options.valueOf("set-quota");
            Set<String> validQuotaTypes = QuotaUtils.validQuotaTypes();
            if (!validQuotaTypes.contains(quotaType)) {
                Utils.croak("Specify a valid quota type from :" + validQuotaTypes);
            }
            if (!options.has("store")) {
                Utils.croak("Must specify the store to enforce the quota on. ");
            }
            if (!options.has("quota-value")) {
                Utils.croak("Must specify the value of the quota being set");
            }
            String storeName = (String) options.valueOf("store");
            String quotaValue = (String) options.valueOf("quota-value");
            executeSetQuota(adminClient, storeName, quotaType, quotaValue);
        } else if (options.has("unset-quota")) {
            String quotaType = (String) options.valueOf("unset-quota");
            Set<String> validQuotaTypes = QuotaUtils.validQuotaTypes();
            if (!validQuotaTypes.contains(quotaType)) {
                Utils.croak("Specify a valid quota type from :" + validQuotaTypes);
            }
            if (!options.has("store")) {
                Utils.croak("Must specify the store to enforce the quota on. ");
            }
            String storeName = (String) options.valueOf("store");
            executeUnsetQuota(adminClient, storeName, quotaType);
        } else if (options.has("get-quota")) {
            String quotaType = (String) options.valueOf("get-quota");
            Set<String> validQuotaTypes = QuotaUtils.validQuotaTypes();
            if (!validQuotaTypes.contains(quotaType)) {
                Utils.croak("Specify a valid quota type from :" + validQuotaTypes);
            }
            if (!options.has("store")) {
                Utils.croak("Must specify the store to enforce the quota on. ");
            }
            String storeName = (String) options.valueOf("store");
            executeGetQuota(adminClient, storeName, quotaType);
        } else {
            Utils.croak("At least one of (delete-partitions, restore, add-node, fetch-entries, " + "fetch-keys, add-stores, delete-store, update-entries, get-metadata, ro-metadata, " + "set-metadata, check-metadata, clear-rebalancing-metadata, async, " + "repair-job, native-backup, rollback, reserve-memory, mirror-url," + " verify-metadata-version, prune-job, purge-slops) must be specified");
        }
    } catch (Exception e) {
        e.printStackTrace();
        Utils.croak(e.getMessage());
    }
}
Also used : VoldemortState(voldemort.store.metadata.MetadataStore.VoldemortState) OptionSet(joptsimple.OptionSet) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) HashSet(java.util.HashSet) Versioned(voldemort.versioning.Versioned) Node(voldemort.cluster.Node) OptionParser(joptsimple.OptionParser) StoreDefinition(voldemort.store.StoreDefinition) StringReader(java.io.StringReader) RebalancerState(voldemort.server.rebalance.RebalancerState) List(java.util.List) ArrayList(java.util.ArrayList) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) Cluster(voldemort.cluster.Cluster) ClusterMapper(voldemort.xml.ClusterMapper) StoreNotFoundException(com.sleepycat.persist.StoreNotFoundException) DecoderException(org.apache.commons.codec.DecoderException) EOFException(java.io.EOFException) SerializationException(voldemort.serialization.SerializationException) FileNotFoundException(java.io.FileNotFoundException) IOException(java.io.IOException) InvalidMetadataException(voldemort.store.InvalidMetadataException) Collection(java.util.Collection) OptionSet(joptsimple.OptionSet) File(java.io.File) AdminClient(voldemort.client.protocol.admin.AdminClient)

Example 17 with ClusterMapper

use of voldemort.xml.ClusterMapper in project voldemort by voldemort.

the class ImportTextDumpToBDB method main.

public static void main(String[] argv) throws Exception {
    OptionParser parser = getParser();
    OptionSet options = parser.parse(argv);
    validateOptions(options);
    String inputPath = (String) options.valueOf("input");
    String storeBdbFolderPath = (String) options.valueOf("bdb");
    String clusterXmlPath = (String) options.valueOf("cluster-xml");
    String storesXmlPath = (String) options.valueOf("stores-xml");
    Integer nodeId = (Integer) options.valueOf("node-id");
    File input = new File(inputPath);
    List<File> dataFiles = new ArrayList<File>();
    if (input.isDirectory()) {
        File[] files = input.listFiles();
        if (files != null)
            Collections.addAll(dataFiles, files);
    } else if (input.isFile()) {
        dataFiles.add(input);
    } else {
        System.err.println(inputPath + "is not file or directory");
    }
    File storeBdbFolder = new File(storeBdbFolderPath);
    final String storeName = storeBdbFolder.getName();
    Cluster cluster = new ClusterMapper().readCluster(new File(clusterXmlPath));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXmlPath));
    StoreDefinition storeDef = null;
    for (StoreDefinition sd : storeDefs) {
        if (sd.getName() != null && sd.getName().equals(storeName)) {
            storeDef = sd;
        }
    }
    if (storeDef == null) {
        throw new VoldemortException("StoreNotfound: " + storeName);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    Properties properties = new Properties();
    properties.put("node.id", "0");
    properties.put("voldemort.home", storeBdbFolder.getParent());
    VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
    voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
    voldemortConfig.setEnableJmx(false);
    voldemortConfig.setBdbOneEnvPerStore(true);
    BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
    class MockStoreDefinition extends StoreDefinition {

        public MockStoreDefinition() {
            super(storeName, null, null, null, null, null, null, null, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        }

        @Override
        public boolean hasMemoryFootprint() {
            return false;
        }
    }
    StoreDefinition mockStoreDef = new MockStoreDefinition();
    StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(mockStoreDef, routingStrategy);
    long reportIntervalMs = 10000L;
    long lastCount = 0;
    long lastInserted = 0;
    Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
    long count = 0;
    long inserted = 0;
    for (File f : dataFiles) {
        try {
            BufferedReader bufferedReader = new BufferedReader(new FileReader(f), READER_BUFFER_SIZE);
            engine.beginBatchModifications();
            while (true) {
                String line = bufferedReader.readLine();
                if (line == null) {
                    break;
                }
                Pair<ByteArray, Versioned<byte[]>> entry;
                try {
                    entry = lineToEntry(line);
                } catch (Exception e) {
                    System.err.println("Skipping line: " + line);
                    e.printStackTrace();
                    continue;
                }
                ByteArray key = entry.getFirst();
                List<Node> nodeList = routingStrategy.routeRequest(key.get());
                for (Node node : nodeList) {
                    if (nodeId == node.getId()) {
                        try {
                            engine.put(key, entry.getSecond(), null);
                            inserted++;
                        } catch (ObsoleteVersionException e) {
                            e.printStackTrace();
                        }
                        break;
                    }
                }
                count++;
                final Long countObject = count;
                final Long insertedObject = inserted;
                Boolean reported = rp.tryReport(new Callable<Boolean>() {

                    @Override
                    public Boolean call() throws Exception {
                        System.out.print(String.format("Imported %15d entries; Inserted %15d entries", countObject, insertedObject));
                        return true;
                    }
                });
                if (reported != null) {
                    long importSpeed = (count - lastCount) / (reportIntervalMs / 1000);
                    long insertSpeed = (inserted - lastInserted) / (reportIntervalMs / 1000);
                    System.out.println(String.format("; ImportSpeed: %8d/s; InsertSpeed: %8d/s ", importSpeed, insertSpeed));
                    lastCount = count;
                    lastInserted = inserted;
                }
            }
            bufferedReader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            engine.endBatchModifications();
        }
    }
    engine.close();
    System.out.println(String.format("Finished importing %d entries (%d inserted, rest discarded)", count, inserted));
}
Also used : Versioned(voldemort.versioning.Versioned) RoutingStrategyFactory(voldemort.routing.RoutingStrategyFactory) Node(voldemort.cluster.Node) OptionParser(joptsimple.OptionParser) VoldemortException(voldemort.VoldemortException) VoldemortConfig(voldemort.server.VoldemortConfig) StoreDefinition(voldemort.store.StoreDefinition) RoutingStrategy(voldemort.routing.RoutingStrategy) ByteArray(voldemort.utils.ByteArray) BdbStorageConfiguration(voldemort.store.bdb.BdbStorageConfiguration) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) Cluster(voldemort.cluster.Cluster) ClusterMapper(voldemort.xml.ClusterMapper) VoldemortException(voldemort.VoldemortException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) DecoderException(org.apache.commons.codec.DecoderException) ObsoleteVersionException(voldemort.versioning.ObsoleteVersionException) OptionSet(joptsimple.OptionSet)

Example 18 with ClusterMapper

use of voldemort.xml.ClusterMapper in project voldemort by voldemort.

the class PartitionAnalysisCLI method main.

public static void main(String[] args) throws Exception {
    setupParser();
    OptionSet options = getValidOptions(args);
    String clusterXML = (String) options.valueOf("cluster");
    String storesXML = (String) options.valueOf("stores");
    Cluster currentCluster = new ClusterMapper().readCluster(new File(clusterXML));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXML));
    PartitionBalance partitionBalance = new PartitionBalance(currentCluster, storeDefs);
    System.out.println(partitionBalance);
}
Also used : StoreDefinition(voldemort.store.StoreDefinition) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) Cluster(voldemort.cluster.Cluster) ClusterMapper(voldemort.xml.ClusterMapper) OptionSet(joptsimple.OptionSet) File(java.io.File)

Example 19 with ClusterMapper

use of voldemort.xml.ClusterMapper in project voldemort by voldemort.

the class RepartitionerCLI method main.

public static void main(String[] args) throws Exception {
    setupParser();
    OptionSet options = getValidOptions(args);
    // Required args
    String currentClusterXML = (String) options.valueOf("current-cluster");
    String currentStoresXML = (String) options.valueOf("current-stores");
    String interimClusterXML = new String(currentClusterXML);
    if (options.has("interim-cluster")) {
        interimClusterXML = (String) options.valueOf("interim-cluster");
    }
    String finalStoresXML = new String(currentStoresXML);
    if (options.has("final-stores")) {
        finalStoresXML = (String) options.valueOf("final-stores");
    }
    Cluster currentCluster = new ClusterMapper().readCluster(new File(currentClusterXML));
    List<StoreDefinition> currentStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(currentStoresXML));
    RebalanceUtils.validateClusterStores(currentCluster, currentStoreDefs);
    Cluster interimCluster = new ClusterMapper().readCluster(new File(interimClusterXML));
    List<StoreDefinition> finalStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(finalStoresXML));
    RebalanceUtils.validateClusterStores(interimCluster, finalStoreDefs);
    RebalanceUtils.validateCurrentInterimCluster(currentCluster, interimCluster);
    // Optional administrivia args
    int attempts = CmdUtils.valueOf(options, "attempts", Repartitioner.DEFAULT_REPARTITION_ATTEMPTS);
    String outputDir = null;
    if (options.has("output-dir")) {
        outputDir = (String) options.valueOf("output-dir");
    }
    // Optional repartitioning args
    boolean disableNodeBalancing = options.has("disable-node-balancing");
    boolean disableZoneBalancing = options.has("disable-zone-balancing");
    boolean enableRandomSwaps = options.has("enable-random-swaps");
    int randomSwapAttempts = CmdUtils.valueOf(options, "random-swap-attempts", Repartitioner.DEFAULT_RANDOM_SWAP_ATTEMPTS);
    int randomSwapSuccesses = CmdUtils.valueOf(options, "random-swap-successes", Repartitioner.DEFAULT_RANDOM_SWAP_SUCCESSES);
    List<Integer> randomSwapZoneIds = CmdUtils.valuesOf(options, "random-swap-zoneids", Repartitioner.DEFAULT_RANDOM_SWAP_ZONE_IDS);
    boolean enableGreedySwaps = options.has("enable-greedy-swaps");
    int greedySwapAttempts = CmdUtils.valueOf(options, "greedy-swap-attempts", Repartitioner.DEFAULT_GREEDY_SWAP_ATTEMPTS);
    int greedyMaxPartitionsPerNode = CmdUtils.valueOf(options, "greedy-max-partitions-per-node", Repartitioner.DEFAULT_GREEDY_MAX_PARTITIONS_PER_NODE);
    int greedyMaxPartitionsPerZone = CmdUtils.valueOf(options, "greedy-max-partitions-per-zone", Repartitioner.DEFAULT_GREEDY_MAX_PARTITIONS_PER_ZONE);
    List<Integer> greedySwapZoneIds = CmdUtils.valuesOf(options, "greedy-swap-zoneids", Repartitioner.DEFAULT_GREEDY_SWAP_ZONE_IDS);
    int maxContiguousPartitionsPerZone = CmdUtils.valueOf(options, "max-contiguous-partitions", Repartitioner.DEFAULT_MAX_CONTIGUOUS_PARTITIONS);
    // Sanity check optional repartitioning args
    if (disableNodeBalancing && !enableRandomSwaps && !enableGreedySwaps && maxContiguousPartitionsPerZone == 0) {
        printUsageAndDie("Did not enable any forms for repartitioning.");
    }
    if ((options.has("random-swap-attempts") || options.has("random-swap-successes")) && !enableRandomSwaps) {
        printUsageAndDie("Provided arguments for generate random swaps but did not enable the feature");
    }
    if ((options.has("greedy-swap-attempts") || options.has("greedy-max-partitions-per-node") || options.has("greedy-max-partitions-per-zone")) && !enableGreedySwaps) {
        printUsageAndDie("Provided arguments for generate greedy swaps but did not enable the feature");
    }
    Repartitioner.repartition(currentCluster, currentStoreDefs, interimCluster, finalStoreDefs, outputDir, attempts, disableNodeBalancing, disableZoneBalancing, enableRandomSwaps, randomSwapAttempts, randomSwapSuccesses, randomSwapZoneIds, enableGreedySwaps, greedySwapAttempts, greedyMaxPartitionsPerNode, greedyMaxPartitionsPerZone, greedySwapZoneIds, maxContiguousPartitionsPerZone);
}
Also used : StoreDefinition(voldemort.store.StoreDefinition) StoreDefinitionsMapper(voldemort.xml.StoreDefinitionsMapper) Cluster(voldemort.cluster.Cluster) ClusterMapper(voldemort.xml.ClusterMapper) OptionSet(joptsimple.OptionSet) File(java.io.File)

Example 20 with ClusterMapper

use of voldemort.xml.ClusterMapper in project voldemort by voldemort.

the class ReplaceNodeCLI method getClusterXML.

private String getClusterXML() {
    ClusterMapper clusterMapper = new ClusterMapper();
    Cluster existingCluster = null;
    String clusterXML = null;
    Map<Integer, String> clusterXMLInNodes = getMetadataXML(MetadataStore.CLUSTER_KEY);
    for (Map.Entry<Integer, String> clusterNodeId : clusterXMLInNodes.entrySet()) {
        String xml = clusterNodeId.getValue();
        Cluster cluster = clusterMapper.readCluster(new StringReader(xml), false);
        if (existingCluster == null) {
            existingCluster = cluster;
            clusterXML = xml;
        } else if (existingCluster.equals(cluster) == false) {
            throw new VoldemortApplicationException("Cluster XMLs are different across nodes, fix that before the node swap...aborting " + clusterNodeId.getKey());
        }
    }
    return clusterXML;
}
Also used : VoldemortApplicationException(voldemort.VoldemortApplicationException) StringReader(java.io.StringReader) Cluster(voldemort.cluster.Cluster) ClusterMapper(voldemort.xml.ClusterMapper) HashMap(java.util.HashMap) Map(java.util.Map)

Aggregations

ClusterMapper (voldemort.xml.ClusterMapper)41 StoreDefinitionsMapper (voldemort.xml.StoreDefinitionsMapper)26 Cluster (voldemort.cluster.Cluster)23 File (java.io.File)18 StoreDefinition (voldemort.store.StoreDefinition)18 StringReader (java.io.StringReader)13 OptionSet (joptsimple.OptionSet)10 Node (voldemort.cluster.Node)10 AdminClient (voldemort.client.protocol.admin.AdminClient)9 VoldemortException (voldemort.VoldemortException)8 Test (org.junit.Test)6 ByteArray (voldemort.utils.ByteArray)6 Versioned (voldemort.versioning.Versioned)6 OptionParser (joptsimple.OptionParser)5 IOException (java.io.IOException)4 InMemoryStorageEngine (voldemort.store.memory.InMemoryStorageEngine)4 FileNotFoundException (java.io.FileNotFoundException)3 ClientConfig (voldemort.client.ClientConfig)3 SocketStoreClientFactory (voldemort.client.SocketStoreClientFactory)3 BufferedReader (java.io.BufferedReader)2