Examples with StoreDefinition voldemort.store.StoreDefinition used on opensource projects

Example 51 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class MetadataStore method readFromDirectory.

public static MetadataStore readFromDirectory(File dir) {
    if (!Utils.isReadableDir(dir))
        throw new IllegalArgumentException("Metadata directory " + dir.getAbsolutePath() + " does not exist or can not be read.");
    String storeDefDirPath = dir.getAbsolutePath() + File.separator + MetadataStore.STORE_DEFINITIONS_STORE_NAME;
    // If config directory does not contain STORES sub directory, then
    // create one by parsing the stores.xml file
    List<String> configurationFiles = Arrays.asList(dir.list());
    if (configurationFiles == null)
        throw new IllegalArgumentException("No configuration found in " + dir.getAbsolutePath() + ".");
    if (!configurationFiles.contains(STORE_DEFINITIONS_STORE_NAME)) {
        // parse stores.xml and create STORES sub-dir
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        List<StoreDefinition> storeDefinitions = null;
        try {
            storeDefinitions = mapper.readStoreList(new File(dir.getAbsolutePath() + File.separator + STORES_KEY));
        } catch (IOException e) {
            throw new VoldemortException("Cannot parse the store definitions from " + STORES_KEY + " file ", e);
        }
        if (storeDefinitions == null) {
            throw new VoldemortException("Neither STORES nor stores.xml exist in the config directory");
        }
        // Create the STORES sub directory
        File storeDefinitionsDir = new File(storeDefDirPath);
        if (!storeDefinitionsDir.mkdir()) {
            throw new VoldemortException("Unable to create " + STORE_DEFINITIONS_STORE_NAME + " sub directory");
        }
        for (StoreDefinition storeDef : storeDefinitions) {
            try {
                FileUtils.writeStringToFile(new File(storeDefDirPath + File.separator + storeDef.getName()), mapper.writeStore(storeDef));
            } catch (IOException e) {
                throw new VoldemortException("Cannot write store definition to file: " + storeDef.getName(), e);
            }
        }
    }
    // Create a STORES configuration engine for STORES sub-directory
    StorageEngine<String, String, String> storesEngine = new ConfigurationStorageEngine(MetadataStore.STORE_DEFINITIONS_STORE_NAME, storeDefDirPath);
    Store<String, String, String> innerStore = new ConfigurationStorageEngine(MetadataStore.METADATA_STORE_NAME, dir.getAbsolutePath());
    MetadataStore store = new MetadataStore(innerStore, storesEngine);
    ServerState state = new ServerState(store);
    JmxUtils.registerMbean(state.getClass().getCanonicalName(), state);
    return store;
}

Example 52 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class MetadataStore method convertObjectToString.

/**
 * Converts Object to byte[] depending on the key
 * <p>
 * StoreRepository takes only StorageEngine<ByteArray,byte[]> and for
 * persistence on disk we need to convert them to String.<br>
 *
 * @param key
 * @param value
 * @return
 */
@SuppressWarnings("unchecked")
private Versioned<String> convertObjectToString(String key, Versioned<Object> value) {
    String valueStr = "";
    if (CLUSTER_KEY.equals(key)) {
        valueStr = clusterMapper.writeCluster((Cluster) value.getValue());
    } else if (STORES_KEY.equals(key)) {
        valueStr = storeMapper.writeStoreList((List<StoreDefinition>) value.getValue());
    } else if (REBALANCING_STEAL_INFO.equals(key)) {
        RebalancerState rebalancerState = (RebalancerState) value.getValue();
        valueStr = rebalancerState.toJsonString();
    } else if (SERVER_STATE_KEY.equals(key) || NODE_ID_KEY.equals(key) || SLOP_STREAMING_ENABLED_KEY.equals(key) || PARTITION_STREAMING_ENABLED_KEY.equals(key) || READONLY_FETCH_ENABLED_KEY.equals(key) || QUOTA_ENFORCEMENT_ENABLED_KEY.equals(key)) {
        valueStr = value.getValue().toString();
    } else if (REBALANCING_SOURCE_CLUSTER_XML.equals(key)) {
        if (value.getValue() != null) {
            valueStr = clusterMapper.writeCluster((Cluster) value.getValue());
        }
    } else if (REBALANCING_SOURCE_STORES_XML.equals(key)) {
        if (value.getValue() != null) {
            valueStr = storeMapper.writeStoreList((List<StoreDefinition>) value.getValue());
        }
    } else if (this.storeNames.contains(key)) {
        valueStr = "<stores>";
        if (value.getValue() != null) {
            valueStr += value.getValue();
        }
        valueStr += "</stores>";
    } else {
        throw new VoldemortException("Unhandled key:'" + key + "' for Object to String serialization.");
    }
    return new Versioned<String>(valueStr, value.getVersion());
}

Example 53 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class MetadataStore method put.

/**
 * helper function to convert strings to bytes as needed.
 *
 * @param key
 * @param value
 */
@SuppressWarnings("unchecked")
public void put(String key, Versioned<Object> value) {
    // acquire write lock
    writeLock.lock();
    try {
        if (this.storeNames.contains(key) || key.equals(STORES_KEY)) {
            // Check for backwards compatibility
            List<StoreDefinition> storeDefinitions = (List<StoreDefinition>) value.getValue();
            StoreDefinitionUtils.validateSchemasAsNeeded(storeDefinitions);
            // If the put is on the entire stores.xml key, delete the
            // additional stores which do not exist in the specified
            // stores.xml
            Set<String> storeNamesToDelete = new HashSet<String>();
            for (String storeName : this.storeNames) {
                storeNamesToDelete.add(storeName);
            }
            // Add / update the list of store definitions specified in the
            // value
            StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
            // Update the STORES directory and the corresponding entry in
            // metadata cache
            Set<String> specifiedStoreNames = new HashSet<String>();
            for (StoreDefinition storeDef : storeDefinitions) {
                specifiedStoreNames.add(storeDef.getName());
                String storeDefStr = mapper.writeStore(storeDef);
                Versioned<String> versionedValueStr = new Versioned<String>(storeDefStr, value.getVersion());
                this.storeDefinitionsStorageEngine.put(storeDef.getName(), versionedValueStr, "");
                // Update the metadata cache
                this.metadataCache.put(storeDef.getName(), new Versioned<Object>(storeDefStr, value.getVersion()));
            }
            if (key.equals(STORES_KEY)) {
                storeNamesToDelete.removeAll(specifiedStoreNames);
                resetStoreDefinitions(storeNamesToDelete);
            }
            // Re-initialize the store definitions
            initStoreDefinitions(value.getVersion());
            // Update routing strategies
            updateRoutingStrategies(getCluster(), getStoreDefList());
        } else if (METADATA_KEYS.contains(key)) {
            // try inserting into inner store first
            putInner(key, convertObjectToString(key, value));
            // cache all keys if innerStore put succeeded
            metadataCache.put(key, value);
            // do special stuff if needed
            if (CLUSTER_KEY.equals(key)) {
                updateRoutingStrategies((Cluster) value.getValue(), getStoreDefList());
            } else if (NODE_ID_KEY.equals(key)) {
                initNodeId(getNodeIdNoLock());
            } else if (SYSTEM_STORES_KEY.equals(key))
                throw new VoldemortException("Cannot overwrite system store definitions");
        } else {
            throw new VoldemortException("Unhandled Key:" + key + " for MetadataStore put()");
        }
    } finally {
        writeLock.unlock();
    }
}

Example 54 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class MetadataStore method createRoutingStrategyMap.

private HashMap<String, RoutingStrategy> createRoutingStrategyMap(Cluster cluster, HashMap<String, StoreDefinition> storeDefs) {
    HashMap<String, RoutingStrategy> map = new HashMap<String, RoutingStrategy>();
    for (StoreDefinition store : storeDefs.values()) {
        map.put(store.getName(), routingFactory.updateRoutingStrategy(store, cluster));
    }
    // add metadata Store route to ALL routing strategy.
    map.put(METADATA_STORE_NAME, new RouteToAllStrategy(getCluster().getNodesShuffled()));
    return map;
}

Example 55 with StoreDefinition

use of voldemort.store.StoreDefinition in project voldemort by voldemort.

the class RequestFileFilter method main.

/**
 * Filter requests specified in a file, generating a new file containing
 * only requests destined for a specific node.
 *
 * @param args See usage for more information
 * @throws Exception In case of I/O or Voldemort-specific errors
 */
public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("node", "[REQUIRED] node id").withRequiredArg().ofType(Integer.class).describedAs("node id");
    parser.accepts("store-name", "[REQUIRED] store name").withRequiredArg().describedAs("store name");
    parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url");
    parser.accepts("input", "[REQUIRED] input request file").withRequiredArg().describedAs("input-file");
    parser.accepts("output", "[REQUIRED] output file").withRequiredArg().describedAs("output-file");
    parser.accepts("string-keys");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "node", "store-name", "url", "input", "output");
    if (missing.size() > 0) {
        System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing));
        parser.printHelpOn(System.err);
        System.exit(1);
    }
    int nodeId = (Integer) options.valueOf("node");
    String storeName = (String) options.valueOf("store-name");
    String bootstrapURL = (String) options.valueOf("url");
    String inputFile = (String) options.valueOf("input");
    String outputFile = (String) options.valueOf("output");
    boolean stringKeys = options.has("string-keys");
    AdminClient adminClient = new AdminClient(bootstrapURL);
    List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId).getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : storeDefinitionList) {
        if (storeName.equals(def.getName())) {
            storeDefinition = def;
        }
    }
    if (storeDefinition == null) {
        Utils.croak("No store found with name\"" + storeName + "\"");
    }
    Cluster cluster = adminClient.metadataMgmtOps.getRemoteCluster(nodeId).getValue();
    Node node = null;
    try {
        node = cluster.getNodeById(nodeId);
    } catch (VoldemortException e) {
        Utils.croak("Can't find a node with id " + nodeId);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    try {
        new RequestFileFilter(storeDefinition, routingStrategy, inputFile, outputFile, node, stringKeys).filter();
    } catch (FileNotFoundException e) {
        Utils.croak(e.getMessage());
    }
}