Search in sources :

Example 51 with GetElements

use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.

the class QueryGeneratorTest method testQueryGeneratorForGetElementsWithEdgeSeeds.

@Test
public void testQueryGeneratorForGetElementsWithEdgeSeeds(@TempDir java.nio.file.Path tempDir) throws IOException, OperationException {
    // Given
    // - Create snapshot folder
    final String folder = String.format("file:///%s", tempDir.toString());
    final String snapshotFolder = folder + "/" + ParquetStore.getSnapshotPath(1000L);
    // - Write out Parquet files so know the partitioning
    CalculatePartitionerTest.writeData(snapshotFolder, new SchemaUtils(schema));
    // - Initialise store
    final ParquetStoreProperties storeProperties = new ParquetStoreProperties();
    storeProperties.setDataDir(folder);
    storeProperties.setTempFilesDir(folder + "/tmpdata");
    final ParquetStore store = (ParquetStore) ParquetStore.createStore("graphId", schema, storeProperties);
    // When 1 - no view, query for edges 0->1, 10--10000, 10000--10 with seed matching type set to EQUAL
    GetElements getElements = new GetElements.Builder().input(new EdgeSeed(0L, 1L, DirectedType.DIRECTED), new EdgeSeed(10L, 1000L, DirectedType.UNDIRECTED), new EdgeSeed(10000L, 10L, DirectedType.EITHER)).seedMatching(SeedMatching.SeedMatchingType.EQUAL).build();
    ParquetQuery query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 1
    final List expected = new ArrayList<>();
    final FilterPredicate source0 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 0L);
    final FilterPredicate source10 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 10L);
    final FilterPredicate source10000 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 10000L);
    final FilterPredicate destination1 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1L);
    final FilterPredicate destination10 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 10L);
    final FilterPredicate destination1000 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1000L);
    final FilterPredicate directedTrue = eq(FilterApi.booleanColumn(ParquetStore.DIRECTED), true);
    final FilterPredicate directedFalse = eq(FilterApi.booleanColumn(ParquetStore.DIRECTED), false);
    final FilterPredicate source0Destination1DirectedTrue = and(and(source0, destination1), directedTrue);
    final FilterPredicate source10Destination1000DirectedFalse = and(and(source10, destination1000), directedFalse);
    final FilterPredicate source10000Destination10DirectedEither = and(source10000, destination10);
    for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        // 0->1 partition 0 of forward
        final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition0File, source0Destination1DirectedTrue, true));
        // 10--1000 partition 1 of forward
        final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition1File, source10Destination1000DirectedFalse, true));
        // 10000--10 partition 9 of forward
        final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition9File, source10000Destination10DirectedEither, true));
    }
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
    // When 2 - no view, query for edges 0->1, 10--10000, 10000--10 with seed matching type set to RELATED
    getElements = new GetElements.Builder().input(new EdgeSeed(0L, 1L, DirectedType.DIRECTED), new EdgeSeed(10L, 1000L, DirectedType.UNDIRECTED), new EdgeSeed(10000L, 10L, DirectedType.EITHER)).seedMatching(SeedMatching.SeedMatchingType.RELATED).build();
    query = new QueryGenerator(store).getParquetQuery(getElements);
    // Then 2
    expected.clear();
    final FilterPredicate vertex0 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 0L);
    final FilterPredicate vertex1 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 1L);
    final FilterPredicate vertex10 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 10L);
    final FilterPredicate vertex1000 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 1000L);
    final FilterPredicate vertex10000 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 10000L);
    final FilterPredicate vertex0or1 = or(vertex0, vertex1);
    final FilterPredicate vertex10or1000 = or(vertex10, vertex1000);
    final FilterPredicate vertex10000or10 = or(vertex10000, vertex10);
    for (final String group : Arrays.asList(TestGroups.ENTITY, TestGroups.ENTITY_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        // 0 and 1 in partition 0
        final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
        expected.add(new ParquetFileQuery(pathForPartition0File, vertex0or1, true));
        // 10 or 1000 and 10000 or 10 in partition 1 (NB 1000 and 10000 't appear in partition 1 but this doesn't cause any incorrect results, and will be fixed in later versions)
        final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
        expected.add(new ParquetFileQuery(pathForPartition1File, or(vertex10or1000, vertex10000or10), true));
        // 10 or 1000 and 1000 or 10000 in partition 9
        final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
        expected.add(new ParquetFileQuery(pathForPartition9File, or(vertex10or1000, vertex10000or10), true));
    }
    for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
        final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
        // 0->1 partition 0 of forward
        final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition0File, source0Destination1DirectedTrue, true));
        // 10--1000 partition 1 of forward
        final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition1File, source10Destination1000DirectedFalse, true));
        // 10000--10 partition 9 of forward
        final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
        // Comment here that don't need to look in the reversed directory
        expected.add(new ParquetFileQuery(pathForPartition9File, source10000Destination10DirectedEither, true));
    }
    assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
}
Also used : ParquetStore(uk.gov.gchq.gaffer.parquetstore.ParquetStore) Path(org.apache.hadoop.fs.Path) ArrayList(java.util.ArrayList) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) SchemaUtils(uk.gov.gchq.gaffer.parquetstore.utils.SchemaUtils) ParquetStoreProperties(uk.gov.gchq.gaffer.parquetstore.ParquetStoreProperties) EdgeSeed(uk.gov.gchq.gaffer.operation.data.EdgeSeed) ArrayList(java.util.ArrayList) List(java.util.List) FilterPredicate(org.apache.parquet.filter2.predicate.FilterPredicate) CalculatePartitionerTest(uk.gov.gchq.gaffer.parquetstore.operation.handler.utilities.CalculatePartitionerTest) LongVertexOperationsTest(uk.gov.gchq.gaffer.parquetstore.operation.handler.LongVertexOperationsTest) Test(org.junit.jupiter.api.Test)

Example 52 with GetElements

use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.

the class GraphTest method shouldManipulateViewRemovingBlacklistedEdgeUsingUpdateViewHook.

@Test
public void shouldManipulateViewRemovingBlacklistedEdgeUsingUpdateViewHook() throws OperationException {
    // Given
    operation = new GetElements.Builder().view(new View.Builder().edge(TestGroups.EDGE_5).edge(TestGroups.EDGE).build()).build();
    final UpdateViewHook updateViewHook = new UpdateViewHook.Builder().blackListElementGroups(Collections.singleton(TestGroups.EDGE)).build();
    given(opChain.getOperations()).willReturn(Lists.newArrayList(operation));
    given(opChain.shallowClone()).willReturn(clonedOpChain);
    given(clonedOpChain.getOperations()).willReturn(Lists.newArrayList(operation));
    given(clonedOpChain.flatten()).willReturn(Arrays.asList(operation));
    final Store store = mock(Store.class);
    given(store.getSchema()).willReturn(new Schema());
    given(store.getProperties()).willReturn(new StoreProperties());
    final Graph graph = new Graph.Builder().config(new GraphConfig.Builder().graphId(GRAPH_ID).addHook(updateViewHook).build()).storeProperties(StreamUtil.storeProps(getClass())).store(store).build();
    final ArgumentCaptor<OperationChain> captor = ArgumentCaptor.forClass(OperationChain.class);
    final ArgumentCaptor<Context> contextCaptor1 = ArgumentCaptor.forClass(Context.class);
    given(store.execute(captor.capture(), contextCaptor1.capture())).willReturn(new ArrayList<>());
    // When / Then
    graph.execute(opChain, user);
    final List<Operation> ops = captor.getValue().getOperations();
    JsonAssert.assertEquals(new View.Builder().edge(TestGroups.EDGE_5).build().toCompactJson(), ((GetElements) ops.get(0)).getView().toCompactJson());
}
Also used : Context(uk.gov.gchq.gaffer.store.Context) Schema(uk.gov.gchq.gaffer.store.schema.Schema) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) Store(uk.gov.gchq.gaffer.store.Store) TestStore(uk.gov.gchq.gaffer.integration.store.TestStore) NamedOperation(uk.gov.gchq.gaffer.named.operation.NamedOperation) Operation(uk.gov.gchq.gaffer.operation.Operation) UpdateViewHook(uk.gov.gchq.gaffer.graph.hook.UpdateViewHook) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) StoreProperties(uk.gov.gchq.gaffer.store.StoreProperties) Test(org.junit.jupiter.api.Test)

Example 53 with GetElements

use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldAddAllOperationsGivenJson.

@Test
public void shouldAddAllOperationsGivenJson() throws IOException {
    // Given
    final byte[] bytes;
    try (final InputStream inputStream = StreamUtil.openStream(getClass(), ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH)) {
        bytes = IOUtils.toByteArray(inputStream);
    }
    final AddOperationsToChain hook = fromJson(bytes);
    Operation discardOutput = new DiscardOutput();
    Operation splitStore = new SplitStoreFromFile();
    Operation validate = new Validate();
    Operation getAdjacentIds = new GetAdjacentIds();
    Operation count = new Count<>();
    Operation countGroups = new CountGroups();
    Operation getElements = new GetElements();
    Operation getAllElements = new GetAllElements();
    Operation limit = new Limit<>();
    final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
    // When
    hook.preExecute(opChain, new Context(new User()));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then(countGroups).then(getElements).then(getAllElements).then(limit).then(validate).then(count).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}
Also used : Context(uk.gov.gchq.gaffer.store.Context) GetAdjacentIds(uk.gov.gchq.gaffer.operation.impl.get.GetAdjacentIds) User(uk.gov.gchq.gaffer.user.User) InputStream(java.io.InputStream) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) Operation(uk.gov.gchq.gaffer.operation.Operation) Count(uk.gov.gchq.gaffer.operation.impl.Count) SplitStoreFromFile(uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile) CountGroups(uk.gov.gchq.gaffer.operation.impl.CountGroups) Validate(uk.gov.gchq.gaffer.operation.impl.Validate) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) DiscardOutput(uk.gov.gchq.gaffer.operation.impl.DiscardOutput) Limit(uk.gov.gchq.gaffer.operation.impl.Limit) Test(org.junit.jupiter.api.Test)

Example 54 with GetElements

use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldAddAllOperationsWithNoAuthsGivenPath.

@Test
public void shouldAddAllOperationsWithNoAuthsGivenPath() throws IOException {
    // Given
    AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
    Operation discardOutput = new DiscardOutput();
    Operation splitStore = new SplitStoreFromFile();
    Operation validate = new Validate();
    Operation getAdjacentIds = new GetAdjacentIds();
    Operation count = new Count<>();
    Operation countGroups = new CountGroups();
    Operation getElements = new GetElements();
    Operation getAllElements = new GetAllElements();
    Operation limit = new Limit<>();
    final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
    // When
    hook.preExecute(opChain, new Context(new User()));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then(countGroups).then(getElements).then(getAllElements).then(limit).then(validate).then(count).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}
Also used : Context(uk.gov.gchq.gaffer.store.Context) GetAdjacentIds(uk.gov.gchq.gaffer.operation.impl.get.GetAdjacentIds) User(uk.gov.gchq.gaffer.user.User) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) Operation(uk.gov.gchq.gaffer.operation.Operation) Count(uk.gov.gchq.gaffer.operation.impl.Count) SplitStoreFromFile(uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile) CountGroups(uk.gov.gchq.gaffer.operation.impl.CountGroups) Validate(uk.gov.gchq.gaffer.operation.impl.Validate) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) DiscardOutput(uk.gov.gchq.gaffer.operation.impl.DiscardOutput) Limit(uk.gov.gchq.gaffer.operation.impl.Limit) Test(org.junit.jupiter.api.Test)

Example 55 with GetElements

use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldAddAllOperationsWithSecondAuthsGivenPath.

@Test
public void shouldAddAllOperationsWithSecondAuthsGivenPath() throws IOException {
    // Given
    AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
    User user = new User.Builder().opAuths("auth2").build();
    Operation splitStore = new SplitStoreFromFile();
    Operation validate = new Validate();
    Operation getAdjacentIds = new GetAdjacentIds();
    Operation countGroups = new CountGroups();
    Operation getElements = new GetElements();
    Operation getAllElements = new GetAllElements();
    final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
    // When
    hook.preExecute(opChain, new Context(user));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(validate).then(getAdjacentIds).then(countGroups).then(getElements).then(getAllElements).then(splitStore).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}
Also used : Context(uk.gov.gchq.gaffer.store.Context) User(uk.gov.gchq.gaffer.user.User) GetAdjacentIds(uk.gov.gchq.gaffer.operation.impl.get.GetAdjacentIds) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) Operation(uk.gov.gchq.gaffer.operation.Operation) SplitStoreFromFile(uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile) CountGroups(uk.gov.gchq.gaffer.operation.impl.CountGroups) Validate(uk.gov.gchq.gaffer.operation.impl.Validate) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) Test(org.junit.jupiter.api.Test)

Aggregations

GetElements (uk.gov.gchq.gaffer.operation.impl.get.GetElements)256 Test (org.junit.jupiter.api.Test)153 EntitySeed (uk.gov.gchq.gaffer.operation.data.EntitySeed)112 User (uk.gov.gchq.gaffer.user.User)102 Element (uk.gov.gchq.gaffer.data.element.Element)91 View (uk.gov.gchq.gaffer.data.elementdefinition.view.View)84 OperationChain (uk.gov.gchq.gaffer.operation.OperationChain)83 AddElements (uk.gov.gchq.gaffer.operation.impl.add.AddElements)67 Context (uk.gov.gchq.gaffer.store.Context)60 Test (org.junit.Test)56 HashSet (java.util.HashSet)52 Graph (uk.gov.gchq.gaffer.graph.Graph)49 ViewElementDefinition (uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition)46 Entity (uk.gov.gchq.gaffer.data.element.Entity)42 ElementFilter (uk.gov.gchq.gaffer.data.element.function.ElementFilter)38 Operation (uk.gov.gchq.gaffer.operation.Operation)35 EdgeSeed (uk.gov.gchq.gaffer.operation.data.EdgeSeed)35 GetAdjacentIds (uk.gov.gchq.gaffer.operation.impl.get.GetAdjacentIds)34 Edge (uk.gov.gchq.gaffer.data.element.Edge)32 ArrayList (java.util.ArrayList)31