Examples with LimitedCloseableIterable uk.gov.gchq.gaffer.commonutil.iterable.LimitedCloseableIterable used on opensource projects

Example 1 with LimitedCloseableIterable

use of uk.gov.gchq.gaffer.commonutil.iterable.LimitedCloseableIterable in project Gaffer by gchq.

the class GetWalksHandler method doOperation.

@Override
public Iterable<Walk> doOperation(final GetWalks getWalks, final Context context, final Store store) throws OperationException {
    // Check input
    if (null == getWalks.getInput()) {
        return null;
    }
    // Check there are some operations
    if (null == getWalks.getOperations()) {
        return new EmptyClosableIterable<>();
    }
    final Integer resultLimit = getWalks.getResultsLimit();
    final int hops = getWalks.getNumberOfGetEdgeOperations();
    final LimitedCloseableIterable limitedInputItr = new LimitedCloseableIterable<>(getWalks.getInput(), 0, resultLimit, false);
    final List<EntityId> originalInput = Lists.newArrayList(limitedInputItr);
    // Check hops and maxHops (if set)
    if (hops == 0) {
        return new EmptyClosableIterable<>();
    } else if (maxHops != null && hops > maxHops) {
        throw new OperationException("GetWalks operation contains " + hops + " hops. The maximum number of hops is: " + maxHops);
    }
    final AdjacencyMaps adjacencyMaps = prune && !getWalks.isIncludePartial() ? new PrunedAdjacencyMaps() : new SimpleAdjacencyMaps();
    final EntityMaps entityMaps = new SimpleEntityMaps();
    List<?> seeds = originalInput;
    // Execute the operations
    for (final OperationChain<Iterable<Element>> operation : getWalks.getOperations()) {
        if (isWhileOperation(operation)) {
            seeds = executeWhileOperation(operation, seeds, resultLimit, context, store, hops, adjacencyMaps, entityMaps);
        } else {
            seeds = executeOperation(operation, seeds, resultLimit, context, store, hops, adjacencyMaps, entityMaps);
        }
    }
    // requested by the user.
    if (entityMaps.size() == adjacencyMaps.size()) {
        entityMaps.add(new EntityMap());
    }
    final GraphWindow graphWindow = new GraphWindow(adjacencyMaps, entityMaps);
    // Track/recombine the edge objects and convert to return type
    final Stream<Walk> walks = Streams.toStream(originalInput).flatMap(seed -> walk(seed.getVertex(), null, graphWindow, new LinkedList<>(), new LinkedList<>(), hops, getWalks.isIncludePartial()).stream());
    return applyConditionalFiltering(walks, getWalks, context, store);
}

Example 2 with LimitedCloseableIterable

use of uk.gov.gchq.gaffer.commonutil.iterable.LimitedCloseableIterable in project Gaffer by gchq.

the class AbstractSampleElementsForSplitPointsHandler method doOperation.

@Override
public List<T> doOperation(final SampleElementsForSplitPoints<T> operation, final Context context, final Store store) throws OperationException {
    final S typedStore = (S) store;
    validate(operation, typedStore);
    final Integer numSplits = getNumSplits(operation, typedStore);
    if (null == numSplits) {
        throw new OperationException("Number of splits is required");
    }
    if (numSplits < 1) {
        return Collections.emptyList();
    }
    final float proportionToSample = operation.getProportionToSample();
    final Random random = new Random(System.currentTimeMillis());
    final Iterable<? extends Element> cleanElements = Iterables.filter(operation.getInput(), e -> null != e && (1 == proportionToSample || random.nextFloat() <= proportionToSample));
    final LimitedCloseableIterable<? extends Element> limitedElements = new LimitedCloseableIterable<>(cleanElements, 0, maxSampledElements, false);
    final Stream<T> recordStream = process(Streams.toStream(limitedElements), typedStore);
    final Stream<T> sortedRecordStream = sort(recordStream, typedStore);
    final List<T> records = sortedRecordStream.collect(Collectors.toList());
    return store.execute(new GenerateSplitPointsFromSample.Builder<T>().input(records).numSplits(numSplits).build(), context);
}

Example 3 with LimitedCloseableIterable

use of uk.gov.gchq.gaffer.commonutil.iterable.LimitedCloseableIterable in project Gaffer by gchq.

the class JoinHandler method doOperation.

@Override
public Iterable<? extends MapTuple> doOperation(final Join<I> operation, final Context context, final Store store) throws OperationException {
    final int limit = operation.getCollectionLimit() != null ? operation.getCollectionLimit() : 100000;
    if (null == operation.getJoinType()) {
        throw new OperationException("A join type must be specified");
    }
    if (null == operation.getMatchMethod()) {
        throw new OperationException("A match method must be supplied");
    }
    if (null == operation.getInput()) {
        operation.setInput(new ArrayList<>());
    }
    MatchKey matchKey = operation.getMatchKey();
    if (null == matchKey) {
        if (!operation.getJoinType().equals(JoinType.INNER)) {
            throw new OperationException("You must specify an Iterable side to match on");
        }
        // setting match key to avoid swapping inputs
        matchKey = MatchKey.LEFT;
    }
    JoinFunction joinFunction = operation.getJoinType().createInstance();
    updateOperationInput(operation.getOperation(), null);
    Iterable<I> rightIterable = (Iterable<I>) getResultsOrNull(operation.getOperation(), context, store);
    final Iterable limitedLeftIterable;
    final Iterable limitedRightIterable;
    try {
        limitedLeftIterable = new LimitedCloseableIterable(operation.getInput(), 0, limit, false);
        limitedRightIterable = new LimitedCloseableIterable(rightIterable, 0, limit, false);
        return joinFunction.join(limitedLeftIterable, limitedRightIterable, operation.getMatchMethod(), matchKey, operation.isFlatten());
    } catch (final LimitExceededException e) {
        throw new OperationException("Join exceeded the collectionLimit, a solution is to increasing collectionLimit value in the join operation.", e);
    }
}