Examples with VoldemortException voldemort.VoldemortException used on opensource projects

Example 1 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class HdfsFetcher method fetchFromSource.

private File fetchFromSource(String sourceFileUrl, String destinationFile, AsyncOperationStatus status, String storeName, long pushVersion, Long diskQuotaSizeInKB, MetadataStore metadataStore) throws Exception {
    ObjectName jmxName = null;
    HdfsCopyStats stats = null;
    FileSystem fs = null;
    sourceFileUrl = VoldemortUtils.modifyURL(sourceFileUrl, voldemortConfig);
    // Flag to indicate whether the fetch is complete or not
    boolean isCompleteFetch = false;
    try {
        // Record as one store fetch
        HdfsCopyStats.storeFetch();
        fs = HadoopUtils.getHadoopFileSystem(voldemortConfig, sourceFileUrl);
        final Path rootPath = new Path(sourceFileUrl);
        File destination = new File(destinationFile);
        if (destination.exists()) {
            throw new VoldemortException("Version directory " + destination.getAbsolutePath() + " already exists");
        }
        boolean isFile = isFile(fs, rootPath);
        stats = new HdfsCopyStats(sourceFileUrl, destination, // stats file initially disabled, to fetch just the first metadata file
        false, maxVersionsStatsFile, isFile, null);
        jmxName = JmxUtils.registerMbean("hdfs-copy-" + copyCount.getAndIncrement(), stats);
        logger.info("Starting fetch for : " + sourceFileUrl);
        FetchStrategy fetchStrategy = new BasicFetchStrategy(this, fs, stats, status, bufferSize);
        if (!isFile) {
            // We are asked to fetch a directory
            Utils.mkdirs(destination);
            HdfsDirectory rootDirectory = new HdfsDirectory(fs, rootPath, this.voldemortConfig);
            List<HdfsDirectory> directoriesToFetch = Lists.newArrayList();
            HdfsFile metadataFile = rootDirectory.getMetadataFile();
            Long expectedDiskSize = -1L;
            if (metadataFile != null) {
                File copyLocation = new File(destination, metadataFile.getPath().getName());
                fetchStrategy.fetch(metadataFile, copyLocation, null);
                rootDirectory.initializeMetadata(copyLocation);
                if (metadataFile.getDiskFileName().equals(ReadOnlyUtils.FULL_STORE_METADATA_FILE)) {
                    // Then we are in build.primary.replica.only mode, and we need to determine which
                    // partition sub-directories to download
                    Set<Integer> partitions = getPartitionsForCurrentNode(metadataStore, storeName);
                    for (int partitionId : partitions) {
                        String partitionKey = ReadOnlyUtils.PARTITION_DIRECTORY_PREFIX + partitionId;
                        ReadOnlyStorageMetadata partitionMetadata = rootDirectory.getMetadata().getNestedMetadata(partitionKey);
                        String diskSizeInBytes = (String) partitionMetadata.get(ReadOnlyStorageMetadata.DISK_SIZE_IN_BYTES);
                        if (diskSizeInBytes != null && diskSizeInBytes != "") {
                            logger.debug("Partition " + partitionId + " is served by this node and is not empty, so it will be downloaded.");
                            if (expectedDiskSize == -1) {
                                expectedDiskSize = Long.parseLong(diskSizeInBytes);
                            } else {
                                expectedDiskSize += Long.parseLong(diskSizeInBytes);
                            }
                            HdfsDirectory partitionDirectory = new HdfsDirectory(fs, new Path(rootPath, partitionKey), this.voldemortConfig);
                            partitionDirectory.initializeMetadata(partitionMetadata);
                            directoriesToFetch.add(partitionDirectory);
                        } else {
                            logger.debug("Partition " + partitionId + " is served by this node but it is empty, so it will be skipped.");
                        }
                    }
                } else {
                    // Then we are not in build.primary.replica.only mode (old behavior), and we
                    // need to download the entire node directory we're currently in.
                    String diskSizeInBytes = (String) rootDirectory.getMetadata().get(ReadOnlyStorageMetadata.DISK_SIZE_IN_BYTES);
                    if (diskSizeInBytes != null && diskSizeInBytes != "") {
                        expectedDiskSize = Long.parseLong(diskSizeInBytes);
                    }
                    directoriesToFetch.add(rootDirectory);
                }
            }
            checkIfQuotaExceeded(diskQuotaSizeInKB, storeName, destination, expectedDiskSize);
            stats = new HdfsCopyStats(sourceFileUrl, destination, enableStatsFile, maxVersionsStatsFile, isFile, new HdfsPathInfo(directoriesToFetch));
            fetchStrategy = new BasicFetchStrategy(this, fs, stats, status, bufferSize);
            logger.debug("directoriesToFetch for store '" + storeName + "': " + Arrays.toString(directoriesToFetch.toArray()));
            for (HdfsDirectory directoryToFetch : directoriesToFetch) {
                Map<HdfsFile, byte[]> fileCheckSumMap = fetchStrategy.fetch(directoryToFetch, destination);
                if (directoryToFetch.validateCheckSum(fileCheckSumMap)) {
                    logger.info("Completed fetch: " + sourceFileUrl);
                } else {
                    stats.checkSumFailed();
                    logger.error("Checksum did not match for " + directoryToFetch.toString() + " !");
                    return null;
                }
            }
            isCompleteFetch = true;
            return destination;
        } else if (allowFetchingOfSingleFile) {
            /**
             * This code path is only used by {@link #main(String[])}
             */
            Utils.mkdirs(destination);
            HdfsFile file = new HdfsFile(fs.getFileStatus(rootPath));
            String fileName = file.getDiskFileName();
            File copyLocation = new File(destination, fileName);
            fetchStrategy.fetch(file, copyLocation, CheckSumType.NONE);
            logger.info("Completed fetch : " + sourceFileUrl);
            isCompleteFetch = true;
            return destination;
        } else {
            logger.error("Source " + rootPath.toString() + " should be a directory");
            return null;
        }
    } catch (Exception e) {
        if (stats != null) {
            stats.reportError("File fetcher failed for destination " + destinationFile, e);
        }
        // Since AuthenticationException may happen before stats object initialization (HadoopUtils.getHadoopFileSystem),
        // we use the static method to capture all the exceptions here.
        HdfsCopyStats.reportExceptionForStats(e);
        if (e instanceof VoldemortException) {
            throw e;
        } else {
            throw new VoldemortException("Error thrown while trying to get data from Hadoop filesystem: " + e.getMessage(), e);
        }
    } finally {
        if (jmxName != null)
            JmxUtils.unregisterMbean(jmxName);
        if (stats != null) {
            stats.complete();
        }
        if (!isCompleteFetch) {
            HdfsCopyStats.incompleteFetch();
        }
        if (fs != null) {
            try {
                fs.close();
            } catch (Exception e) {
                String errorMessage = "Caught " + e.getClass().getSimpleName() + " while trying to close the filesystem instance (harmless).";
                if (stats != null) {
                    stats.reportError(errorMessage, e);
                }
                logger.debug(errorMessage, e);
            }
        }
    }
}

Example 2 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class AvroStoreBuilderMapper method map.

/**
 * Create the voldemort key and value from the input Avro record by
 * extracting the key and value and map it out for each of the responsible
 * voldemort nodes
 *
 * The output value is the node_id & partition_id of the responsible node
 * followed by serialized value
 */
@Override
public void map(GenericData.Record record, AvroCollector<Pair<ByteBuffer, ByteBuffer>> collector, Reporter reporter) throws IOException {
    byte[] keyBytes = null;
    byte[] valBytes = null;
    Object keyRecord = null;
    Object valRecord = null;
    try {
        keyRecord = record.get(keyField);
        valRecord = record.get(valField);
        keyBytes = keySerializer.toBytes(keyRecord);
        valBytes = valueSerializer.toBytes(valRecord);
        this.collectorWrapper.setCollector(collector);
        this.mapper.map(keyBytes, valBytes, this.collectorWrapper);
        recordCounter++;
    } catch (OutOfMemoryError oom) {
        logger.error(oomErrorMessage(reporter));
        if (keyBytes == null) {
            logger.error("keyRecord caused OOM!");
        } else {
            logger.error("keyRecord: " + keyRecord);
            logger.error("valRecord: " + (valBytes == null ? "caused OOM" : valRecord));
        }
        throw new VoldemortException(oomErrorMessage(reporter), oom);
    }
}

Example 3 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class HadoopStoreWriter method write.

@Override
public void write(BytesWritable key, Iterator<BytesWritable> iterator, Reporter reporter) throws IOException {
    // Read chunk id
    int chunkId = ReadOnlyUtils.chunk(key.getBytes(), getNumChunks());
    initFileStreams(chunkId);
    // Write key and position
    this.indexFileStream[chunkId].write(key.getBytes(), 0, key.getLength());
    this.indexFileSizeInBytes[chunkId] += key.getLength();
    this.indexFileStream[chunkId].writeInt(this.position[chunkId]);
    this.indexFileSizeInBytes[chunkId] += ByteUtils.SIZE_OF_INT;
    // Run key through checksum digest
    if (this.checkSumDigestIndex[chunkId] != null) {
        this.checkSumDigestIndex[chunkId].update(key.getBytes(), 0, key.getLength());
        this.checkSumDigestIndex[chunkId].update(this.position[chunkId]);
    }
    short numTuples = 0;
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    DataOutputStream valueStream = new DataOutputStream(stream);
    while (iterator.hasNext()) {
        BytesWritable writable = iterator.next();
        byte[] valueBytes = writable.getBytes();
        int offsetTillNow = 0;
        /**
         * Below, we read the node id, partition id and replica type of each record
         * coming in, and validate that it is consistent with the other IDs seen so
         * far. This is to catch potential regressions to the shuffling logic in:
         *
         * {@link AbstractStoreBuilderConfigurable#getPartition(byte[], byte[], int)}
         */
        // Read node Id
        int currentNodeId = ByteUtils.readInt(valueBytes, offsetTillNow);
        if (this.nodeId == -1) {
            this.nodeId = currentNodeId;
        } else if (this.nodeId != currentNodeId) {
            throw new IllegalArgumentException("Should not get various nodeId shuffled to us! " + "First nodeId seen: " + this.nodeId + ", currentNodeId: " + currentNodeId);
        }
        offsetTillNow += ByteUtils.SIZE_OF_INT;
        // Read partition id
        int currentPartitionId = ByteUtils.readInt(valueBytes, offsetTillNow);
        if (this.partitionId == -1) {
            this.partitionId = currentPartitionId;
        } else if (this.partitionId != currentPartitionId) {
            throw new IllegalArgumentException("Should not get various partitionId shuffled to us! " + "First partitionId seen: " + this.partitionId + ", currentPartitionId: " + currentPartitionId);
        }
        offsetTillNow += ByteUtils.SIZE_OF_INT;
        // Read replica type
        if (getSaveKeys()) {
            int currentReplicaType = (int) ByteUtils.readBytes(valueBytes, offsetTillNow, ByteUtils.SIZE_OF_BYTE);
            if (this.replicaType == -1) {
                this.replicaType = currentReplicaType;
            } else if (this.replicaType != currentReplicaType) {
                throw new IllegalArgumentException("Should not get various replicaType shuffled to us! " + "First replicaType seen: " + this.replicaType + ", currentReplicaType: " + currentReplicaType);
            }
            if (getBuildPrimaryReplicasOnly() && this.replicaType > 0) {
                throw new IllegalArgumentException("Should not get any replicaType > 0 shuffled to us" + " when buildPrimaryReplicasOnly mode is enabled!");
            }
            offsetTillNow += ByteUtils.SIZE_OF_BYTE;
        }
        int valueLength = writable.getLength() - offsetTillNow;
        if (getSaveKeys()) {
            // Write ( key_length, value_length, key, value )
            valueStream.write(valueBytes, offsetTillNow, valueLength);
        } else {
            // Write (value_length + value)
            valueStream.writeInt(valueLength);
            valueStream.write(valueBytes, offsetTillNow, valueLength);
        }
        numTuples++;
        // malicious ( We obviously expect collisions when we save keys )
        if (!getSaveKeys() && numTuples > 1)
            throw new VoldemortException("Duplicate keys detected for md5 sum " + ByteUtils.toHexString(ByteUtils.copy(key.getBytes(), 0, key.getLength())));
    }
    if (numTuples < 0) {
        // Overflow
        throw new VoldemortException("Found too many collisions: chunk " + chunkId + " has exceeded " + MAX_HASH_COLLISIONS + " collisions.");
    } else if (numTuples > 1) {
        // Update number of collisions + max keys per collision
        reporter.incrCounter(CollisionCounter.NUM_COLLISIONS, 1);
        long numCollisions = reporter.getCounter(CollisionCounter.MAX_COLLISIONS).getCounter();
        if (numTuples > numCollisions) {
            reporter.incrCounter(CollisionCounter.MAX_COLLISIONS, numTuples - numCollisions);
        }
    }
    // Flush the value
    valueStream.flush();
    byte[] value = stream.toByteArray();
    // First, if save keys flag set the number of keys
    if (getSaveKeys()) {
        this.valueFileStream[chunkId].writeShort(numTuples);
        this.valueFileSizeInBytes[chunkId] += ByteUtils.SIZE_OF_SHORT;
        this.position[chunkId] += ByteUtils.SIZE_OF_SHORT;
        if (this.checkSumDigestValue[chunkId] != null) {
            this.checkSumDigestValue[chunkId].update(numTuples);
        }
    }
    this.valueFileStream[chunkId].write(value);
    this.valueFileSizeInBytes[chunkId] += value.length;
    this.position[chunkId] += value.length;
    if (this.checkSumDigestValue[chunkId] != null) {
        this.checkSumDigestValue[chunkId].update(value);
    }
    if (this.position[chunkId] < 0)
        throw new VoldemortException("Chunk overflow exception: chunk " + chunkId + " has exceeded " + MAX_CHUNK_SIZE + " bytes.");
}

Example 4 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class AbstractStoreBuilderConfigurable method configure.

public void configure(JobConf conf) {
    this.cluster = new ClusterMapper().readCluster(new StringReader(conf.get("cluster.xml")));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new StringReader(conf.get("stores.xml")));
    if (storeDefs.size() != 1)
        throw new IllegalStateException("Expected to find only a single store, but found multiple!");
    this.storeDef = storeDefs.get(0);
    this.numChunks = conf.getInt(NUM_CHUNKS, -1);
    if (this.numChunks < 1) {
        // be overridden by the user.
        throw new VoldemortException(NUM_CHUNKS + " not specified in the MapReduce JobConf (should NEVER happen)");
    }
    this.saveKeys = conf.getBoolean(VoldemortBuildAndPushJob.SAVE_KEYS, true);
    this.reducerPerBucket = conf.getBoolean(VoldemortBuildAndPushJob.REDUCER_PER_BUCKET, true);
    this.buildPrimaryReplicasOnly = conf.getBoolean(VoldemortBuildAndPushJob.BUILD_PRIMARY_REPLICAS_ONLY, false);
    if (buildPrimaryReplicasOnly && !saveKeys) {
        throw new IllegalStateException(VoldemortBuildAndPushJob.BUILD_PRIMARY_REPLICAS_ONLY + " can only be true if " + VoldemortBuildAndPushJob.SAVE_KEYS + " is also true.");
    }
}

Example 5 with VoldemortException

use of voldemort.VoldemortException in project voldemort by voldemort.

the class AbstractStoreBuilderConfigurable method getPartition.

/**
 * This function computes which reduce task to shuffle a record to.
 */
public int getPartition(byte[] key, byte[] value, int numReduceTasks) {
    try {
        /**
         * {@link partitionId} is the Voldemort primary partition that this
         * record belongs to.
         */
        int partitionId = ByteUtils.readInt(value, ByteUtils.SIZE_OF_INT);
        /**
         * This is the base number we will ultimately mod by {@link numReduceTasks}
         * to determine which reduce task to shuffle to.
         */
        int magicNumber = partitionId;
        if (getSaveKeys() && !buildPrimaryReplicasOnly) {
            /**
             * When saveKeys is enabled (which also implies we are generating
             * READ_ONLY_V2 format files), then we are generating files with
             * a replica type, with one file per replica.
             *
             * Each replica is sent to a different reducer, and thus the
             * {@link magicNumber} is scaled accordingly.
             *
             * The downside to this is that it is pretty wasteful. The files
             * generated for each replicas are identical to one another, so
             * there's no point in generating them independently in many
             * reducers.
             *
             * This is one of the reasons why buildPrimaryReplicasOnly was
             * written. In this mode, we only generate the files for the
             * primary replica, which means the number of reducers is
             * minimized and {@link magicNumber} does not need to be scaled.
             */
            int replicaType = (int) ByteUtils.readBytes(value, 2 * ByteUtils.SIZE_OF_INT, ByteUtils.SIZE_OF_BYTE);
            magicNumber = magicNumber * getStoreDef().getReplicationFactor() + replicaType;
        }
        if (!getReducerPerBucket()) {
            /**
             * Partition files can be split in many chunks in order to limit the
             * maximum file size downloaded and handled by Voldemort servers.
             *
             * {@link chunkId} represents which chunk of partition then current
             * record belongs to.
             */
            int chunkId = ReadOnlyUtils.chunk(key, getNumChunks());
            /**
             * When reducerPerBucket is disabled, all chunks are sent to a
             * different reducer. This increases parallelism at the expense
             * of adding more load on Hadoop.
             *
             * {@link magicNumber} is thus scaled accordingly, in order to
             * leverage the extra reducers available to us.
             */
            magicNumber = magicNumber * getNumChunks() + chunkId;
        }
        /**
         * Finally, we mod {@link magicNumber} by {@link numReduceTasks},
         * since the MapReduce framework expects the return of this function
         * to be bounded by the number of reduce tasks running in the job.
         */
        return magicNumber % numReduceTasks;
    } catch (Exception e) {
        throw new VoldemortException("Caught exception in getPartition()!" + " key: " + ByteUtils.toHexString(key) + ", value: " + ByteUtils.toHexString(value) + ", numReduceTasks: " + numReduceTasks, e);
    }
}