Examples with ChainedIterable uk.gov.gchq.gaffer.commonutil.iterable.ChainedIterable used on opensource projects

Example 6 with ChainedIterable

use of uk.gov.gchq.gaffer.commonutil.iterable.ChainedIterable in project Gaffer by gchq.

the class RoadTrafficStringElementGenerator method _apply.

@Override
public Iterable<Element> _apply(final String line) {
    final String[] fields = extractFields(line);
    if (null == fields) {
        return Collections.emptyList();
    }
    // Extract required fields
    final FreqMap vehicleCountsByType = getVehicleCounts(fields);
    final Date startDate = getDate(fields[dCount.ordinal()], fields[Hour.ordinal()]);
    final Date endDate = null != startDate ? DateUtils.addMilliseconds(DateUtils.addHours(startDate, 1), -1) : null;
    final String region = fields[Region_Name.ordinal()];
    final String location = fields[ONS_LA_Name.ordinal()];
    final String road = fields[Road.ordinal()];
    final String junctionA = road + ":" + fields[A_Junction.ordinal()];
    final String junctionB = road + ":" + fields[B_Junction.ordinal()];
    final String junctionALocation = fields[A_Ref_E.ordinal()] + "," + fields[A_Ref_N.ordinal()];
    final String junctionBLocation = fields[B_Ref_E.ordinal()] + "," + fields[B_Ref_N.ordinal()];
    final List<Edge> edges = Arrays.asList(new Edge.Builder().group(ElementGroup.REGION_CONTAINS_LOCATION).source(region).dest(location).directed(true).build(), new Edge.Builder().group(ElementGroup.LOCATION_CONTAINS_ROAD).source(location).dest(road).directed(true).build(), new Edge.Builder().group(ElementGroup.ROAD_HAS_JUNCTION).source(road).dest(junctionA).directed(true).build(), new Edge.Builder().group(ElementGroup.ROAD_HAS_JUNCTION).source(road).dest(junctionB).directed(true).build(), new Edge.Builder().group(ElementGroup.JUNCTION_LOCATED_AT).source(junctionA).dest(junctionALocation).directed(true).build(), new Edge.Builder().group(ElementGroup.JUNCTION_LOCATED_AT).source(junctionB).dest(junctionBLocation).directed(true).build(), new Edge.Builder().group(ElementGroup.ROAD_USE).source(junctionA).dest(junctionB).directed(true).property("startDate", startDate).property("endDate", endDate).property("count", getTotalCount(vehicleCountsByType)).property("countByVehicleType", vehicleCountsByType).build());
    final List<Entity> entities = Arrays.asList(new Entity.Builder().group(ElementGroup.JUNCTION_USE).vertex(junctionA).property("countByVehicleType", vehicleCountsByType).property("endDate", endDate).property("startDate", startDate).property("count", getTotalCount(vehicleCountsByType)).build(), new Entity.Builder().group(ElementGroup.JUNCTION_USE).vertex(junctionB).property("countByVehicleType", vehicleCountsByType).property("endDate", endDate).property("startDate", startDate).property("count", getTotalCount(vehicleCountsByType)).build());
    final List<Entity> cardinalityEntities = createCardinalities(edges);
    return new ChainedIterable<>(edges, entities, cardinalityEntities);
}

Example 7 with ChainedIterable

use of uk.gov.gchq.gaffer.commonutil.iterable.ChainedIterable in project Gaffer by gchq.

the class AggregatorUtil method queryAggregate.

/**
 * Applies query time aggregation to the provided iterable of {@link Element}s.
 * This uses the groupBy properties in the provided {@link View} or {@link Schema} to group
 * the elements prior to aggregating them. Aggregation of Edges can optionally be
 * configured to include the Matched Vertex field.
 * <p>
 * NOTE - this is done in memory so the size of the iterable should be limited.
 *
 * @param elements             the elements to be aggregated
 * @param schema               the schema containing the aggregators and groupBy properties to use
 * @param view                 the view containing the aggregators and groupBy properties to use
 * @param includeMatchedVertex whether aggregation groups should include the Edge Matched Vertex
 * @return the aggregated elements.
 */
public static CloseableIterable<Element> queryAggregate(final Iterable<? extends Element> elements, final Schema schema, final View view, final boolean includeMatchedVertex) {
    if (null == schema) {
        throw new IllegalArgumentException("Schema is required");
    }
    if (null == view) {
        throw new IllegalArgumentException("View is required");
    }
    final Collection<String> aggregatedGroups = schema.getAggregatedGroups();
    final List<Element> aggregatableElements = new ArrayList<>();
    final List<Element> nonAggregatedElements = new ArrayList<>();
    for (final Element element : elements) {
        if (null != element) {
            if (aggregatedGroups.contains(element.getGroup())) {
                aggregatableElements.add(element);
            } else {
                nonAggregatedElements.add(element);
            }
        }
    }
    final Iterable<Element> aggregatedElements = Streams.toStream(aggregatableElements).collect(Collectors.groupingBy(new ToQueryElementKey(schema, view, includeMatchedVertex), Collectors.reducing(null, new QueryElementBinaryOperator(schema, view)))).values();
    return new ChainedIterable<>(aggregatedElements, nonAggregatedElements);
}

Example 8 with ChainedIterable

use of uk.gov.gchq.gaffer.commonutil.iterable.ChainedIterable in project Gaffer by gchq.

the class AggregatorUtil method ingestAggregate.

/**
 * Applies ingest aggregation to the provided iterable of {@link Element}s.
 * This uses the groupBy properties in the provided {@link Schema} to group
 * the elements prior to aggregating them.
 * <p>
 * NOTE - this is done in memory so the size of the iterable should be limited.
 *
 * @param elements the elements to be aggregated
 * @param schema   the schema containing the aggregators and groupBy properties to use
 * @return the aggregated elements.
 */
public static CloseableIterable<Element> ingestAggregate(final Iterable<? extends Element> elements, final Schema schema) {
    if (null == schema) {
        throw new IllegalArgumentException("Schema is required");
    }
    final Collection<String> aggregatedGroups = schema.getAggregatedGroups();
    final List<Element> aggregatableElements = new ArrayList<>();
    final List<Element> nonAggregatedElements = new ArrayList<>();
    for (final Element element : elements) {
        if (null != element) {
            if (aggregatedGroups.contains(element.getGroup())) {
                aggregatableElements.add(element);
            } else {
                nonAggregatedElements.add(element);
            }
        }
    }
    final Iterable<Element> aggregatedElements = Streams.toStream(aggregatableElements).collect(Collectors.groupingBy(new ToIngestElementKey(schema), Collectors.reducing(null, new IngestElementBinaryOperator(schema)))).values();
    return new ChainedIterable<>(aggregatedElements, nonAggregatedElements);
}

Example 9 with ChainedIterable

use of uk.gov.gchq.gaffer.commonutil.iterable.ChainedIterable in project Gaffer by gchq.

the class FederatedGetAdjacentIdsHandlerTest method validateMergeResultsFromFieldObjects.

@Override
protected boolean validateMergeResultsFromFieldObjects(final CloseableIterable<? extends EntityId> result, final Object... resultParts) {
    assertNotNull(result);
    final Iterable[] resultPartItrs = Arrays.copyOf(resultParts, resultParts.length, Iterable[].class);
    final ArrayList<Object> elements = Lists.newArrayList(new ChainedIterable<>(resultPartItrs));
    int i = 0;
    for (EntityId e : result) {
        assertTrue(e instanceof Entity);
        elements.contains(e);
        i++;
    }
    assertEquals(elements.size(), i);
    return true;
}