Examples with Partition org.apache.spark.Partition used on opensource projects

Example 6 with Partition

use of org.apache.spark.Partition in project Gaffer by gchq.

the class RFileReaderIterator method init.

private void init() throws IOException {
    final AccumuloTablet accumuloTablet = (AccumuloTablet) partition;
    LOGGER.info("Initialising RFileReaderIterator for files {}", StringUtils.join(accumuloTablet.getFiles(), ','));
    final AccumuloConfiguration accumuloConfiguration = SiteConfiguration.getInstance();
    // Required column families according to the configuration
    final Set<ByteSequence> requiredColumnFamilies = InputConfigurator.getFetchedColumns(AccumuloInputFormat.class, configuration).stream().map(Pair::getFirst).map(c -> new ArrayByteSequence(c.toString())).collect(Collectors.toSet());
    LOGGER.info("RFileReaderIterator will read column families of {}", StringUtils.join(requiredColumnFamilies, ','));
    // Column families
    final List<SortedKeyValueIterator<Key, Value>> iterators = new ArrayList<>();
    for (final String filename : accumuloTablet.getFiles()) {
        final Path path = new Path(filename);
        final FileSystem fs = path.getFileSystem(configuration);
        final RFile.Reader rFileReader = new RFile.Reader(new CachableBlockFile.Reader(fs, path, configuration, null, null, accumuloConfiguration));
        iterators.add(rFileReader);
    }
    mergedIterator = new MultiIterator(iterators, true);
    // Apply visibility filtering iterator
    if (null != auths) {
        final Authorizations authorizations = new Authorizations(auths.toArray(new String[auths.size()]));
        final SortedKeyValueIterator<Key, Value> visibilityFilter = VisibilityFilter.wrap(mergedIterator, authorizations, new byte[] {});
        final IteratorSetting visibilityIteratorSetting = new IteratorSetting(1, "auth", VisibilityFilter.class);
        visibilityFilter.init(mergedIterator, visibilityIteratorSetting.getOptions(), null);
        iteratorAfterIterators = visibilityFilter;
        LOGGER.info("Set authorizations to {}", authorizations);
    } else {
        iteratorAfterIterators = mergedIterator;
    }
    // Apply iterator stack
    final List<IteratorSetting> iteratorSettings = getIteratorSettings();
    iteratorSettings.sort(Comparator.comparingInt(IteratorSetting::getPriority));
    for (final IteratorSetting is : iteratorSettings) {
        iteratorAfterIterators = applyIterator(iteratorAfterIterators, is);
    }
    taskContext.addTaskCompletionListener(context -> close());
    final Range range = new Range(accumuloTablet.getStartRow(), true, accumuloTablet.getEndRow(), false);
    iteratorAfterIterators.seek(range, requiredColumnFamilies, true);
    LOGGER.info("Initialised iterator");
}