Examples with ElementSeed uk.gov.gchq.gaffer.operation.data.ElementSeed used on opensource projects

Example 31 with ElementSeed

use of uk.gov.gchq.gaffer.operation.data.ElementSeed in project Gaffer by gchq.

the class BloomFilterIT method seek.

private void seek(final FileSKVIterator reader, final EntitySeed seed, final RangeFactory rangeFactory) throws IOException, RangeFactoryException {
    final View view = new View.Builder().edge(TestGroups.EDGE).entity(TestGroups.ENTITY).build();
    final GetElements<ElementSeed, ?> operation = new GetElements<>(view);
    final List<Range> range = rangeFactory.getRange(seed, operation);
    for (final Range ran : range) {
        reader.seek(ran, new ArrayList<ByteSequence>(), false);
    }
}

Example 32 with ElementSeed

use of uk.gov.gchq.gaffer.operation.data.ElementSeed in project Gaffer by gchq.

the class StoreTest method shouldHandleMultiStepOperations.

@Test
public void shouldHandleMultiStepOperations() throws Exception {
    // Given
    final Schema schema = createSchemaMock();
    final StoreProperties properties = mock(StoreProperties.class);
    final StoreImpl store = new StoreImpl();
    final CloseableIterable<Element> getElementsResult = mock(CloseableIterable.class);
    final AddElements addElements1 = new AddElements();
    final GetElements<ElementSeed, Element> getElements = new GetElements<>();
    final OperationChain<CloseableIterable<Element>> opChain = new OperationChain.Builder().first(addElements1).then(getElements).build();
    given(addElementsHandler.doOperation(addElements1, context, store)).willReturn(null);
    given(getElementsHandler.doOperation(getElements, context, store)).willReturn(getElementsResult);
    store.initialise(schema, properties);
    // When
    final CloseableIterable<Element> result = store.execute(opChain, user);
    // Then
    assertSame(getElementsResult, result);
}

Example 33 with ElementSeed

use of uk.gov.gchq.gaffer.operation.data.ElementSeed in project Gaffer by gchq.

the class RowIdAggregatorTest method testAggregatingMultiplePropertySetsAcrossRowIDRange.

private void testAggregatingMultiplePropertySetsAcrossRowIDRange(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException, RangeFactoryException {
    String visibilityString = "public";
    try {
        // Create table
        // (this method creates the table, removes the versioning iterator, and adds the SetOfStatisticsCombiner iterator).
        TableUtils.createTable(store);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties2 = new Properties();
        properties2.put(AccumuloPropertyNames.COUNT, 1);
        final Properties properties3 = new Properties();
        properties3.put(AccumuloPropertyNames.COUNT, 2);
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("2");
        edge.setDestination("1");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("B");
        edge2.setDestination("Z");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 1);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("3");
        edge3.setDestination("8");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge6 = new Edge("BasicEdge2");
        edge6.setSource("1");
        edge6.setDestination("5");
        edge6.setDirected(true);
        edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge7 = new Edge("BasicEdge2");
        edge7.setSource("2");
        edge7.setDestination("6");
        edge7.setDirected(true);
        edge7.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge7.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge7.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge8 = new Edge("BasicEdge2");
        edge8.setSource("4");
        edge8.setDestination("8");
        edge8.setDirected(true);
        edge8.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge8.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge8.putProperty(AccumuloPropertyNames.PROP_4, 0);
        final Edge edge9 = new Edge("BasicEdge2");
        edge9.setSource("5");
        edge9.setDestination("9");
        edge9.setDirected(true);
        edge9.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge9.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge9.putProperty(AccumuloPropertyNames.PROP_4, 0);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        final Key key4 = elementConverter.getKeysFromEdge(edge6).getFirst();
        final Key key5 = elementConverter.getKeysFromEdge(edge7).getFirst();
        final Key key6 = elementConverter.getKeysFromEdge(edge8).getFirst();
        final Key key7 = elementConverter.getKeysFromEdge(edge9).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties2);
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties3);
        final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties1);
        final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties2);
        //Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        final Mutation m6 = new Mutation(key4.getRow());
        m6.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
        final Mutation m7 = new Mutation(key5.getRow());
        m7.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
        final Mutation m8 = new Mutation(key6.getRow());
        m8.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value5);
        final Mutation m9 = new Mutation(key7.getRow());
        m9.put(key7.getColumnFamily(), key7.getColumnQualifier(), new ColumnVisibility(key7.getColumnVisibility()), key7.getTimestamp(), value5);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.addMutation(m6);
        writer.addMutation(m7);
        writer.addMutation(m8);
        writer.addMutation(m9);
        writer.close();
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final BatchScanner scanner = store.getConnection().createBatchScanner(store.getProperties().getTable(), authorizations, 1000);
        try {
            scanner.addScanIterator(store.getKeyPackage().getIteratorFactory().getRowIDAggregatorIteratorSetting(store, "BasicEdge2"));
        } catch (IteratorSettingException e) {
            fail(e.getMessage());
        }
        final RangeFactory rangeF = store.getKeyPackage().getRangeFactory();
        final Range r = rangeF.getRangeFromPair(new Pair<ElementSeed>((new EntitySeed("1")), new EntitySeed("4")), new SummariseGroupOverRanges());
        final Range r2 = rangeF.getRangeFromPair(new Pair<ElementSeed>((new EntitySeed("5")), new EntitySeed("5")), new SummariseGroupOverRanges());
        scanner.setRanges(Arrays.asList(r, r2));
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        Entry<Key, Value> entry = it.next();
        Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        Edge expectedEdge = new Edge("BasicEdge2");
        expectedEdge.setSource("4");
        expectedEdge.setDestination("8");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 3);
        assertEquals(expectedEdge, readEdge);
        assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check we get the Result of the second provided range
        assertTrue(it.hasNext());
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        expectedEdge = new Edge("BasicEdge2");
        expectedEdge.setSource("5");
        expectedEdge.setDestination("9");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
        assertEquals(expectedEdge, readEdge);
        //Check no additional rows are found. (For a table of this size we shouldn't see this)
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableExistsException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 34 with ElementSeed

use of uk.gov.gchq.gaffer.operation.data.ElementSeed in project Gaffer by gchq.

the class GetElementsinRangesHandlerTest method shouldSummariseOutGoingEdgesOnly.

private void shouldSummariseOutGoingEdgesOnly(final AccumuloStore store) throws OperationException {
    // Create set to query for
    final Set<Pair<ElementSeed>> simpleEntityRanges = new HashSet<>();
    //get Everything between 0 and 1 (Note we are using strings and string serialisers, with this ordering 0999 is before 1)
    simpleEntityRanges.add(new Pair<ElementSeed>(new EntitySeed("0"), new EntitySeed("C")));
    final View view = new View.Builder(defaultView).entity(TestGroups.ENTITY, new ViewElementDefinition.Builder().groupBy().build()).edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build();
    final GetElementsInRanges<Pair<ElementSeed>, Element> operation = new GetElementsInRanges<>(view, simpleEntityRanges);
    //All Edges stored should be outgoing from our provided seeds.
    operation.setIncludeIncomingOutGoing(IncludeIncomingOutgoingType.OUTGOING);
    final GetElementsInRangesHandler handler = new GetElementsInRangesHandler();
    final CloseableIterable<Element> rangeElements = handler.doOperation(operation, user, store);
    int count = 0;
    for (final Element elm : rangeElements) {
        //Make sure every element has been summarised
        assertEquals(9, elm.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        count++;
    }
    assertEquals(1000, count);
    rangeElements.close();
    simpleEntityRanges.clear();
    //This should get everything between 0 and 0799 (again being string ordering 0800 is more than 08)
    simpleEntityRanges.add(new Pair<ElementSeed>(new EntitySeed("0"), new EntitySeed("08")));
    final CloseableIterable<Element> elements = handler.doOperation(operation, user, store);
    count = 0;
    for (final Element elm : elements) {
        //Make sure every element has been summarised
        assertEquals(9, elm.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        count++;
    }
    assertEquals(800, count);
    elements.close();
}

Example 35 with ElementSeed

use of uk.gov.gchq.gaffer.operation.data.ElementSeed in project Gaffer by gchq.

the class GetElementsinRangesHandlerTest method shouldReturnElementsNoSummarisation.

private void shouldReturnElementsNoSummarisation(final AccumuloStore store) throws OperationException {
    // Create set to query for
    final Set<Pair<ElementSeed>> simpleEntityRanges = new HashSet<>();
    final User user = new User();
    //get Everything between 0 and 1 (Note we are using strings and string serialisers, with this ordering 0999 is before 1)
    simpleEntityRanges.add(new Pair<ElementSeed>(new EntitySeed("0"), new EntitySeed("1")));
    final GetElementsInRanges<Pair<ElementSeed>, Element> operation = new GetElementsInRanges<>(defaultView, simpleEntityRanges);
    final GetElementsInRangesHandler handler = new GetElementsInRangesHandler();
    CloseableIterable<Element> elementsInRanges = handler.doOperation(operation, user, store);
    final int elementsInRangesCount = Iterables.size(elementsInRanges);
    //Each Edge was put in 3 times with different col qualifiers, without summarisation we expect this number
    assertEquals(1000 * 3, elementsInRangesCount);
    elementsInRanges.close();
    simpleEntityRanges.clear();
    //This should get everything between 0 and 0799 (again being string ordering 0800 is more than 08)
    simpleEntityRanges.add(new Pair<ElementSeed>(new EntitySeed("0"), new EntitySeed("08")));
    final CloseableIterable<Element> elements = handler.doOperation(operation, user, store);
    final int count = Iterables.size(elements);
    //Each Edge was put in 3 times with different col qualifiers, without summarisation we expect this number
    assertEquals(800 * 3, count);
    elements.close();
}