Examples with PartitionInfo alluxio.thrift.PartitionInfo used on opensource projects

Example 1 with PartitionInfo

use of alluxio.thrift.PartitionInfo in project alluxio by Alluxio.

the class KeyValueInputFormat method getSplits.

/**
   * Returns a list of {@link KeyValueInputSplit} where each split is one key-value partition.
   *
   * @param jobContext MapReduce job configuration
   * @return list of {@link InputSplit}s, each split is a partition
   * @throws IOException if information about the partition cannot be retrieved
   */
@Override
public List<InputSplit> getSplits(JobContext jobContext) throws IOException {
    // The paths are MapReduce program's inputs specified in
    // {@code mapreduce.input.fileinputformat.inputdir}, each path should be a key-value store.
    Path[] paths = FileInputFormat.getInputPaths(jobContext);
    List<InputSplit> splits = new ArrayList<>();
    try {
        for (Path path : paths) {
            List<PartitionInfo> partitionInfos = mKeyValueMasterClient.getPartitionInfo(new AlluxioURI(path.toString()));
            for (PartitionInfo partitionInfo : partitionInfos) {
                splits.add(new KeyValueInputSplit(partitionInfo));
            }
        }
    } catch (AlluxioException e) {
        throw new IOException(e);
    }
    return splits;
}

Example 2 with PartitionInfo

use of alluxio.thrift.PartitionInfo in project alluxio by Alluxio.

the class BaseKeyValueStoreReader method get.

@Override
public ByteBuffer get(ByteBuffer key) throws IOException, AlluxioException {
    Preconditions.checkNotNull(key);
    int left = 0;
    int right = mPartitions.size();
    while (left < right) {
        int middle = (right + left) / 2;
        PartitionInfo partition = mPartitions.get(middle);
        // NOTE: keyStart and keyLimit are both inclusive
        if (key.compareTo(partition.bufferForKeyStart()) < 0) {
            right = middle;
        } else if (key.compareTo(partition.bufferForKeyLimit()) > 0) {
            left = middle + 1;
        } else {
            // The key is either in this partition or not in the key-value store
            long blockId = partition.getBlockId();
            try (KeyValuePartitionReader reader = KeyValuePartitionReader.Factory.create(blockId)) {
                return reader.get(key);
            }
        }
    }
    return null;
}

Example 3 with PartitionInfo

use of alluxio.thrift.PartitionInfo in project alluxio by Alluxio.

the class BaseKeyValueStoreWriter method completePartition.

/**
   * Completes the current partition.
   *
   * @throws IOException if non-Alluxio error occurs
   * @throws AlluxioException if Alluxio error occurs
   */
private void completePartition() throws IOException, AlluxioException {
    if (mWriter == null) {
        return;
    }
    mWriter.close();
    List<Long> blockIds = mFileSystem.getStatus(getPartitionName()).getBlockIds();
    long blockId = blockIds.get(0);
    PartitionInfo info = new PartitionInfo(mKeyStart, mKeyLimit, blockId, mWriter.keyCount());
    mMasterClient.completePartition(mStoreUri, info);
    mPartitionIndex++;
}

Example 4 with PartitionInfo

use of alluxio.thrift.PartitionInfo in project alluxio by Alluxio.

the class KeyValueMaster method completePartitionFromEntry.

// Marks a partition complete, called when replaying journals
private void completePartitionFromEntry(CompletePartitionEntry entry) throws FileDoesNotExistException {
    PartitionInfo info = new PartitionInfo(entry.getKeyStartBytes().asReadOnlyByteBuffer(), entry.getKeyLimitBytes().asReadOnlyByteBuffer(), entry.getBlockId(), entry.getKeyCount());
    completePartitionInternal(entry.getStoreId(), info);
}

Example 5 with PartitionInfo

use of alluxio.thrift.PartitionInfo in project alluxio by Alluxio.

the class KeyValueMaster method streamToJournalCheckpoint.

@Override
public synchronized void streamToJournalCheckpoint(JournalOutputStream outputStream) throws IOException {
    for (Map.Entry<Long, List<PartitionInfo>> entry : mCompleteStoreToPartitions.entrySet()) {
        long fileId = entry.getKey();
        List<PartitionInfo> partitions = entry.getValue();
        outputStream.writeEntry(newCreateStoreEntry(fileId));
        for (PartitionInfo info : partitions) {
            outputStream.writeEntry(newCompletePartitionEntry(fileId, info));
        }
        outputStream.writeEntry(newCompleteStoreEntry(fileId));
    }
    for (Map.Entry<Long, List<PartitionInfo>> entry : mIncompleteStoreToPartitions.entrySet()) {
        long fileId = entry.getKey();
        List<PartitionInfo> partitions = entry.getValue();
        outputStream.writeEntry(newCreateStoreEntry(fileId));
        for (PartitionInfo info : partitions) {
            outputStream.writeEntry(newCompletePartitionEntry(fileId, info));
        }
    }
}