use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.
the class QueryGeneratorTest method testQueryGeneratorForGetElementsWithEdgeSeeds.
@Test
public void testQueryGeneratorForGetElementsWithEdgeSeeds(@TempDir java.nio.file.Path tempDir) throws IOException, OperationException {
// Given
// - Create snapshot folder
final String folder = String.format("file:///%s", tempDir.toString());
final String snapshotFolder = folder + "/" + ParquetStore.getSnapshotPath(1000L);
// - Write out Parquet files so know the partitioning
CalculatePartitionerTest.writeData(snapshotFolder, new SchemaUtils(schema));
// - Initialise store
final ParquetStoreProperties storeProperties = new ParquetStoreProperties();
storeProperties.setDataDir(folder);
storeProperties.setTempFilesDir(folder + "/tmpdata");
final ParquetStore store = (ParquetStore) ParquetStore.createStore("graphId", schema, storeProperties);
// When 1 - no view, query for edges 0->1, 10--10000, 10000--10 with seed matching type set to EQUAL
GetElements getElements = new GetElements.Builder().input(new EdgeSeed(0L, 1L, DirectedType.DIRECTED), new EdgeSeed(10L, 1000L, DirectedType.UNDIRECTED), new EdgeSeed(10000L, 10L, DirectedType.EITHER)).seedMatching(SeedMatching.SeedMatchingType.EQUAL).build();
ParquetQuery query = new QueryGenerator(store).getParquetQuery(getElements);
// Then 1
final List expected = new ArrayList<>();
final FilterPredicate source0 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 0L);
final FilterPredicate source10 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 10L);
final FilterPredicate source10000 = eq(FilterApi.longColumn(ParquetStore.SOURCE), 10000L);
final FilterPredicate destination1 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1L);
final FilterPredicate destination10 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 10L);
final FilterPredicate destination1000 = eq(FilterApi.longColumn(ParquetStore.DESTINATION), 1000L);
final FilterPredicate directedTrue = eq(FilterApi.booleanColumn(ParquetStore.DIRECTED), true);
final FilterPredicate directedFalse = eq(FilterApi.booleanColumn(ParquetStore.DIRECTED), false);
final FilterPredicate source0Destination1DirectedTrue = and(and(source0, destination1), directedTrue);
final FilterPredicate source10Destination1000DirectedFalse = and(and(source10, destination1000), directedFalse);
final FilterPredicate source10000Destination10DirectedEither = and(source10000, destination10);
for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
// 0->1 partition 0 of forward
final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
// Comment here that don't need to look in the reversed directory
expected.add(new ParquetFileQuery(pathForPartition0File, source0Destination1DirectedTrue, true));
// 10--1000 partition 1 of forward
final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
// Comment here that don't need to look in the reversed directory
expected.add(new ParquetFileQuery(pathForPartition1File, source10Destination1000DirectedFalse, true));
// 10000--10 partition 9 of forward
final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
// Comment here that don't need to look in the reversed directory
expected.add(new ParquetFileQuery(pathForPartition9File, source10000Destination10DirectedEither, true));
}
assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
// When 2 - no view, query for edges 0->1, 10--10000, 10000--10 with seed matching type set to RELATED
getElements = new GetElements.Builder().input(new EdgeSeed(0L, 1L, DirectedType.DIRECTED), new EdgeSeed(10L, 1000L, DirectedType.UNDIRECTED), new EdgeSeed(10000L, 10L, DirectedType.EITHER)).seedMatching(SeedMatching.SeedMatchingType.RELATED).build();
query = new QueryGenerator(store).getParquetQuery(getElements);
// Then 2
expected.clear();
final FilterPredicate vertex0 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 0L);
final FilterPredicate vertex1 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 1L);
final FilterPredicate vertex10 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 10L);
final FilterPredicate vertex1000 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 1000L);
final FilterPredicate vertex10000 = eq(FilterApi.longColumn(ParquetStore.VERTEX), 10000L);
final FilterPredicate vertex0or1 = or(vertex0, vertex1);
final FilterPredicate vertex10or1000 = or(vertex10, vertex1000);
final FilterPredicate vertex10000or10 = or(vertex10000, vertex10);
for (final String group : Arrays.asList(TestGroups.ENTITY, TestGroups.ENTITY_2)) {
final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
// 0 and 1 in partition 0
final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
expected.add(new ParquetFileQuery(pathForPartition0File, vertex0or1, true));
// 10 or 1000 and 10000 or 10 in partition 1 (NB 1000 and 10000 't appear in partition 1 but this doesn't cause any incorrect results, and will be fixed in later versions)
final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
expected.add(new ParquetFileQuery(pathForPartition1File, or(vertex10or1000, vertex10000or10), true));
// 10 or 1000 and 1000 or 10000 in partition 9
final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
expected.add(new ParquetFileQuery(pathForPartition9File, or(vertex10or1000, vertex10000or10), true));
}
for (final String group : Arrays.asList(TestGroups.EDGE, TestGroups.EDGE_2)) {
final Path groupFolderPath = new Path(snapshotFolder, ParquetStore.getGroupSubDir(group, false));
// 0->1 partition 0 of forward
final Path pathForPartition0File = new Path(groupFolderPath, ParquetStore.getFile(0));
// Comment here that don't need to look in the reversed directory
expected.add(new ParquetFileQuery(pathForPartition0File, source0Destination1DirectedTrue, true));
// 10--1000 partition 1 of forward
final Path pathForPartition1File = new Path(groupFolderPath, ParquetStore.getFile(1));
// Comment here that don't need to look in the reversed directory
expected.add(new ParquetFileQuery(pathForPartition1File, source10Destination1000DirectedFalse, true));
// 10000--10 partition 9 of forward
final Path pathForPartition9File = new Path(groupFolderPath, ParquetStore.getFile(9));
// Comment here that don't need to look in the reversed directory
expected.add(new ParquetFileQuery(pathForPartition9File, source10000Destination10DirectedEither, true));
}
assertThat(expected).containsOnly(query.getAllParquetFileQueries().toArray());
}
use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.
the class GraphTest method shouldManipulateViewRemovingBlacklistedEdgeUsingUpdateViewHook.
@Test
public void shouldManipulateViewRemovingBlacklistedEdgeUsingUpdateViewHook() throws OperationException {
// Given
operation = new GetElements.Builder().view(new View.Builder().edge(TestGroups.EDGE_5).edge(TestGroups.EDGE).build()).build();
final UpdateViewHook updateViewHook = new UpdateViewHook.Builder().blackListElementGroups(Collections.singleton(TestGroups.EDGE)).build();
given(opChain.getOperations()).willReturn(Lists.newArrayList(operation));
given(opChain.shallowClone()).willReturn(clonedOpChain);
given(clonedOpChain.getOperations()).willReturn(Lists.newArrayList(operation));
given(clonedOpChain.flatten()).willReturn(Arrays.asList(operation));
final Store store = mock(Store.class);
given(store.getSchema()).willReturn(new Schema());
given(store.getProperties()).willReturn(new StoreProperties());
final Graph graph = new Graph.Builder().config(new GraphConfig.Builder().graphId(GRAPH_ID).addHook(updateViewHook).build()).storeProperties(StreamUtil.storeProps(getClass())).store(store).build();
final ArgumentCaptor<OperationChain> captor = ArgumentCaptor.forClass(OperationChain.class);
final ArgumentCaptor<Context> contextCaptor1 = ArgumentCaptor.forClass(Context.class);
given(store.execute(captor.capture(), contextCaptor1.capture())).willReturn(new ArrayList<>());
// When / Then
graph.execute(opChain, user);
final List<Operation> ops = captor.getValue().getOperations();
JsonAssert.assertEquals(new View.Builder().edge(TestGroups.EDGE_5).build().toCompactJson(), ((GetElements) ops.get(0)).getView().toCompactJson());
}
use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.
the class AddOperationsToChainTest method shouldAddAllOperationsGivenJson.
@Test
public void shouldAddAllOperationsGivenJson() throws IOException {
// Given
final byte[] bytes;
try (final InputStream inputStream = StreamUtil.openStream(getClass(), ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH)) {
bytes = IOUtils.toByteArray(inputStream);
}
final AddOperationsToChain hook = fromJson(bytes);
Operation discardOutput = new DiscardOutput();
Operation splitStore = new SplitStoreFromFile();
Operation validate = new Validate();
Operation getAdjacentIds = new GetAdjacentIds();
Operation count = new Count<>();
Operation countGroups = new CountGroups();
Operation getElements = new GetElements();
Operation getAllElements = new GetAllElements();
Operation limit = new Limit<>();
final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
// When
hook.preExecute(opChain, new Context(new User()));
// Then
final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then(countGroups).then(getElements).then(getAllElements).then(limit).then(validate).then(count).build();
JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}
use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.
the class AddOperationsToChainTest method shouldAddAllOperationsWithNoAuthsGivenPath.
@Test
public void shouldAddAllOperationsWithNoAuthsGivenPath() throws IOException {
// Given
AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
Operation discardOutput = new DiscardOutput();
Operation splitStore = new SplitStoreFromFile();
Operation validate = new Validate();
Operation getAdjacentIds = new GetAdjacentIds();
Operation count = new Count<>();
Operation countGroups = new CountGroups();
Operation getElements = new GetElements();
Operation getAllElements = new GetAllElements();
Operation limit = new Limit<>();
final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
// When
hook.preExecute(opChain, new Context(new User()));
// Then
final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then(countGroups).then(getElements).then(getAllElements).then(limit).then(validate).then(count).build();
JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}
use of uk.gov.gchq.gaffer.operation.impl.get.GetElements in project Gaffer by gchq.
the class AddOperationsToChainTest method shouldAddAllOperationsWithSecondAuthsGivenPath.
@Test
public void shouldAddAllOperationsWithSecondAuthsGivenPath() throws IOException {
// Given
AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
User user = new User.Builder().opAuths("auth2").build();
Operation splitStore = new SplitStoreFromFile();
Operation validate = new Validate();
Operation getAdjacentIds = new GetAdjacentIds();
Operation countGroups = new CountGroups();
Operation getElements = new GetElements();
Operation getAllElements = new GetAllElements();
final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
// When
hook.preExecute(opChain, new Context(user));
// Then
final OperationChain expectedOpChain = new OperationChain.Builder().first(validate).then(getAdjacentIds).then(countGroups).then(getElements).then(getAllElements).then(splitStore).build();
JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}
Aggregations