Examples with VoldemortException voldemort.VoldemortException used on opensource projects

Example 21 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class MetadataStore method readFromDirectory.

public static MetadataStore readFromDirectory(File dir) {
    if (!Utils.isReadableDir(dir))
        throw new IllegalArgumentException("Metadata directory " + dir.getAbsolutePath() + " does not exist or can not be read.");
    String storeDefDirPath = dir.getAbsolutePath() + File.separator + MetadataStore.STORE_DEFINITIONS_STORE_NAME;
    // If config directory does not contain STORES sub directory, then
    // create one by parsing the stores.xml file
    List<String> configurationFiles = Arrays.asList(dir.list());
    if (configurationFiles == null)
        throw new IllegalArgumentException("No configuration found in " + dir.getAbsolutePath() + ".");
    if (!configurationFiles.contains(STORE_DEFINITIONS_STORE_NAME)) {
        // parse stores.xml and create STORES sub-dir
        StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
        List<StoreDefinition> storeDefinitions = null;
        try {
            storeDefinitions = mapper.readStoreList(new File(dir.getAbsolutePath() + File.separator + STORES_KEY));
        } catch (IOException e) {
            throw new VoldemortException("Cannot parse the store definitions from " + STORES_KEY + " file ", e);
        }
        if (storeDefinitions == null) {
            throw new VoldemortException("Neither STORES nor stores.xml exist in the config directory");
        }
        // Create the STORES sub directory
        File storeDefinitionsDir = new File(storeDefDirPath);
        if (!storeDefinitionsDir.mkdir()) {
            throw new VoldemortException("Unable to create " + STORE_DEFINITIONS_STORE_NAME + " sub directory");
        }
        for (StoreDefinition storeDef : storeDefinitions) {
            try {
                FileUtils.writeStringToFile(new File(storeDefDirPath + File.separator + storeDef.getName()), mapper.writeStore(storeDef));
            } catch (IOException e) {
                throw new VoldemortException("Cannot write store definition to file: " + storeDef.getName(), e);
            }
        }
    }
    // Create a STORES configuration engine for STORES sub-directory
    StorageEngine<String, String, String> storesEngine = new ConfigurationStorageEngine(MetadataStore.STORE_DEFINITIONS_STORE_NAME, storeDefDirPath);
    Store<String, String, String> innerStore = new ConfigurationStorageEngine(MetadataStore.METADATA_STORE_NAME, dir.getAbsolutePath());
    MetadataStore store = new MetadataStore(innerStore, storesEngine);
    ServerState state = new ServerState(store);
    JmxUtils.registerMbean(state.getClass().getCanonicalName(), state);
    return store;
}

Example 22 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class MetadataStore method put.

/**
     * helper function to convert strings to bytes as needed.
     * 
     * @param key
     * @param value
     */
@SuppressWarnings("unchecked")
public void put(String key, Versioned<Object> value) {
    // acquire write lock
    writeLock.lock();
    try {
        if (this.storeNames.contains(key) || key.equals(STORES_KEY)) {
            // Check for backwards compatibility
            List<StoreDefinition> storeDefinitions = (List<StoreDefinition>) value.getValue();
            StoreDefinitionUtils.validateSchemasAsNeeded(storeDefinitions);
            // If the put is on the entire stores.xml key, delete the
            // additional stores which do not exist in the specified
            // stores.xml
            Set<String> storeNamesToDelete = new HashSet<String>();
            for (String storeName : this.storeNames) {
                storeNamesToDelete.add(storeName);
            }
            // Add / update the list of store definitions specified in the
            // value
            StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
            // Update the STORES directory and the corresponding entry in
            // metadata cache
            Set<String> specifiedStoreNames = new HashSet<String>();
            for (StoreDefinition storeDef : storeDefinitions) {
                specifiedStoreNames.add(storeDef.getName());
                String storeDefStr = mapper.writeStore(storeDef);
                Versioned<String> versionedValueStr = new Versioned<String>(storeDefStr, value.getVersion());
                this.storeDefinitionsStorageEngine.put(storeDef.getName(), versionedValueStr, "");
                // Update the metadata cache
                this.metadataCache.put(storeDef.getName(), new Versioned<Object>(storeDefStr, value.getVersion()));
            }
            if (key.equals(STORES_KEY)) {
                storeNamesToDelete.removeAll(specifiedStoreNames);
                resetStoreDefinitions(storeNamesToDelete);
            }
            // Re-initialize the store definitions
            initStoreDefinitions(value.getVersion());
            // Update routing strategies
            updateRoutingStrategies(getCluster(), getStoreDefList());
        } else if (METADATA_KEYS.contains(key)) {
            // try inserting into inner store first
            putInner(key, convertObjectToString(key, value));
            // cache all keys if innerStore put succeeded
            metadataCache.put(key, value);
            // do special stuff if needed
            if (CLUSTER_KEY.equals(key)) {
                updateRoutingStrategies((Cluster) value.getValue(), getStoreDefList());
            } else if (NODE_ID_KEY.equals(key)) {
                initNodeId(getNodeIdNoLock());
            } else if (SYSTEM_STORES_KEY.equals(key))
                throw new VoldemortException("Cannot overwrite system store definitions");
        } else {
            throw new VoldemortException("Unhandled Key:" + key + " for MetadataStore put()");
        }
    } finally {
        writeLock.unlock();
    }
}

Example 23 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class MetadataStore method validate.

@SuppressWarnings("unchecked")
public void validate(ByteArray keyBytes, Versioned<byte[]> valueBytes, byte[] transforms) throws VoldemortException {
    String key = ByteUtils.getString(keyBytes.get(), "UTF-8");
    Versioned<String> value = new Versioned<String>(ByteUtils.getString(valueBytes.getValue(), "UTF-8"), valueBytes.getVersion());
    Versioned<Object> valueObject = convertStringToObject(key, value);
    if (key.equals(MetadataStore.STORES_KEY)) {
        List<StoreDefinition> storeDefinitions = (List<StoreDefinition>) valueObject.getValue();
        Set<String> existingStores = new HashSet<String>(this.storeNames);
        Set<String> specifiedStoreNames = new HashSet<String>();
        for (StoreDefinition storeDef : storeDefinitions) {
            String storeName = storeDef.getName();
            if (specifiedStoreNames.contains(storeName)) {
                throw new VoldemortException(" Duplicate store names in Stores.xml for storeName " + storeName);
            }
            specifiedStoreNames.add(storeName);
        }
        existingStores.removeAll(specifiedStoreNames);
        // adding it back to allow again.
        if (existingStores.size() > 0) {
            logger.warn(" Set metadata does not support store deletion. This will leave the store in an " + "inconsistent state. Stores (Inconsistent) missing in set metadata " + Arrays.toString(existingStores.toArray()));
        }
        specifiedStoreNames.removeAll(this.storeNames);
        if (specifiedStoreNames.size() > 0) {
            logger.warn(" Set metadata does not support store addition . This will leave the store in an " + "inconsistent state. Stores (Inconsistent) added in set metadata " + Arrays.toString(specifiedStoreNames.toArray()));
        }
    }
}

Example 24 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class ChunkedFileSet method renameReadOnlyV2Files.

/**
     * This function looks for files with the "wrong" replica type in their name, and
     * if it finds any, renames them.
     *
     * Those files may have ended up on this server either because:
     *  - 1. We restored them from another server, where they were named according to
     *       another replica type. Or,
     *  - 2. The {@link voldemort.store.readonly.mr.azkaban.VoldemortBuildAndPushJob}
     *       and the {@link voldemort.store.readonly.fetcher.HdfsFetcher} are
     *       operating in 'build.primary.replicas.only' mode, so they only ever built
     *       and fetched replica 0 of any given file.
     *
     * Note: This is an implementation detail of the READONLY_V2 naming scheme, and should
     * not be used outside of that scope.
     *
     * @param masterPartitionId partition ID of the "primary replica"
     * @param correctReplicaType replica number which should be found on the current
     *                           node for the provided masterPartitionId.
     */
private void renameReadOnlyV2Files(int masterPartitionId, int correctReplicaType) {
    for (int replica = 0; replica < routingStrategy.getNumReplicas(); replica++) {
        if (replica != correctReplicaType) {
            int chunkId = 0;
            while (true) {
                String fileName = Integer.toString(masterPartitionId) + "_" + Integer.toString(replica) + "_" + Integer.toString(chunkId);
                File index = getIndexFile(fileName);
                File data = getDataFile(fileName);
                if (index.exists() && data.exists()) {
                    // We found files with the "wrong" replica type, so let's rename them
                    String correctFileName = Integer.toString(masterPartitionId) + "_" + Integer.toString(correctReplicaType) + "_" + Integer.toString(chunkId);
                    File indexWithCorrectReplicaType = getIndexFile(correctFileName);
                    File dataWithCorrectReplicaType = getDataFile(correctFileName);
                    Utils.move(index, indexWithCorrectReplicaType);
                    Utils.move(data, dataWithCorrectReplicaType);
                    // Maybe change this to DEBUG?
                    logger.info("Renamed files with wrong replica type: " + index.getAbsolutePath() + "|data -> " + indexWithCorrectReplicaType.getName() + "|data");
                } else if (index.exists() ^ data.exists()) {
                    throw new VoldemortException("One of the following does not exist: " + index.toString() + " or " + data.toString() + ".");
                } else {
                    // so let's move on.
                    break;
                }
                chunkId++;
            }
        }
    }
}

Example 25 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class ChunkedFileSet method initVersion2.

public void initVersion2() {
    int globalChunkId = 0;
    if (this.nodePartitionIds != null) {
        // Go over every node in the cluster
        for (Node node : routingStrategy.getNodes()) {
            // For each node, go over every partition mastered by that node
            for (int masterPartitionId : node.getPartitionIds()) {
                // For each master partition, determine whether one of its replicas needs to be
                // hosted by the current node, and if yes, take care of its index/data files.
                int correctReplicaType = getReplicaTypeForPartition(masterPartitionId);
                if (correctReplicaType == -1) {
                    // node at all (i.e.: we host none of its replicas), so we skip to the next partition.
                    continue;
                }
                renameReadOnlyV2Files(masterPartitionId, correctReplicaType);
                // Generate bucket information for <master partition ID, replica type>
                Pair<Integer, Integer> bucket = Pair.create(masterPartitionId, correctReplicaType);
                int chunkId = 0;
                while (true) {
                    // Loop over all the chunks in this bucket
                    String fileName = Integer.toString(bucket.getFirst()) + "_" + Integer.toString(bucket.getSecond()) + "_" + Integer.toString(chunkId);
                    File index = getIndexFile(fileName);
                    File data = getDataFile(fileName);
                    if (!index.exists() && !data.exists()) {
                        if (chunkId == 0) {
                            // partition
                            try {
                                index.createNewFile();
                                data.createNewFile();
                                logger.info("No index or data files found, creating empty files for partition " + masterPartitionId + " and replicating partition " + correctReplicaType);
                            } catch (IOException e) {
                                throw new VoldemortException("Error creating empty read-only files for partition " + masterPartitionId + " and replicating partition " + correctReplicaType, e);
                            }
                        } else {
                            break;
                        }
                    } else if (index.exists() ^ data.exists()) {
                        throw new VoldemortException("One of the following does not exist: " + index.toString() + " or " + data.toString() + ".");
                    }
                    if (chunkId == 0) {
                        chunkIdToChunkStart.put(bucket, globalChunkId);
                    }
                    /* Deal with file sizes */
                    long indexLength = index.length();
                    long dataLength = data.length();
                    validateFileSizes(indexLength, dataLength);
                    indexFileSizes.add((int) indexLength);
                    dataFileSizes.add((int) dataLength);
                    fileNames.add(fileName);
                    /* Add the file channel for data */
                    dataFiles.add(openChannel(data));
                    mapAndRememberIndexFile(index);
                    chunkId++;
                    globalChunkId++;
                }
                chunkIdToNumChunks.put(bucket, chunkId);
            }
        }
        if (indexFileSizes.size() == 0)
            throw new VoldemortException("No chunk files found in directory " + baseDir.toString());
    }
}