Examples with Operation uk.gov.gchq.gaffer.operation.Operation used on opensource projects

Example 36 with Operation

use of uk.gov.gchq.gaffer.operation.Operation in project Gaffer by gchq.

the class AccumuloStore method updateConfiguration.

/**
 * Updates a Hadoop {@link Configuration} with information needed to connect to the Accumulo store. It adds
 * iterators to apply the provided {@link View}. This method will be used by operations that run MapReduce
 * or Spark jobs against the Accumulo store.
 *
 * @param conf         A {@link Configuration} to be updated.
 * @param graphFilters The operation {@link GraphFilters} to be applied.
 * @param user         The {@link User} to be used.
 * @throws StoreException If there is a failure to connect to Accumulo or a problem setting the iterators.
 */
public void updateConfiguration(final Configuration conf, final GraphFilters graphFilters, final User user) throws StoreException {
    try {
        final View view = graphFilters.getView();
        // Table name
        LOGGER.info("Updating configuration with table name of {}", getTableName());
        InputConfigurator.setInputTableName(AccumuloInputFormat.class, conf, getTableName());
        // User
        addUserToConfiguration(conf);
        // Authorizations
        Authorizations authorisations;
        if (null != user && null != user.getDataAuths()) {
            authorisations = new Authorizations(user.getDataAuths().toArray(new String[user.getDataAuths().size()]));
        } else {
            authorisations = new Authorizations();
        }
        InputConfigurator.setScanAuthorizations(AccumuloInputFormat.class, conf, authorisations);
        LOGGER.info("Updating configuration with authorizations of {}", authorisations);
        // Zookeeper
        addZookeeperToConfiguration(conf);
        // Add keypackage, schema and view to conf
        conf.set(ElementInputFormat.KEY_PACKAGE, getProperties().getKeyPackageClass());
        LOGGER.info("Updating configuration with key package of {}", getProperties().getKeyPackageClass());
        conf.set(ElementInputFormat.SCHEMA, new String(getSchema().toCompactJson(), CommonConstants.UTF_8));
        LOGGER.debug("Updating configuration with Schema of {}", getSchema());
        conf.set(ElementInputFormat.VIEW, new String(view.toCompactJson(), CommonConstants.UTF_8));
        LOGGER.debug("Updating configuration with View of {}", view);
        if (view.hasGroups()) {
            // Add the columns to fetch
            final Collection<org.apache.accumulo.core.util.Pair<Text, Text>> columnFamilyColumnQualifierPairs = Stream.concat(view.getEntityGroups().stream(), view.getEdgeGroups().stream()).map(g -> new org.apache.accumulo.core.util.Pair<>(new Text(g), (Text) null)).collect(Collectors.toSet());
            InputConfigurator.fetchColumns(AccumuloInputFormat.class, conf, columnFamilyColumnQualifierPairs);
            LOGGER.info("Updated configuration with column family/qualifiers of {}", StringUtils.join(columnFamilyColumnQualifierPairs, ','));
            // Add iterators that depend on the view
            final IteratorSetting elementPreFilter = getKeyPackage().getIteratorFactory().getElementPreAggregationFilterIteratorSetting(view, this);
            if (null != elementPreFilter) {
                InputConfigurator.addIterator(AccumuloInputFormat.class, conf, elementPreFilter);
                LOGGER.info("Added pre-aggregation filter iterator of {}", elementPreFilter);
            }
            final IteratorSetting elementPostFilter = getKeyPackage().getIteratorFactory().getElementPostAggregationFilterIteratorSetting(view, this);
            if (null != elementPostFilter) {
                InputConfigurator.addIterator(AccumuloInputFormat.class, conf, elementPostFilter);
                LOGGER.info("Added post-aggregation filter iterator of {}", elementPostFilter);
            }
            final IteratorSetting edgeEntityDirFilter = getKeyPackage().getIteratorFactory().getEdgeEntityDirectionFilterIteratorSetting(graphFilters);
            if (null != edgeEntityDirFilter) {
                InputConfigurator.addIterator(AccumuloInputFormat.class, conf, edgeEntityDirFilter);
                LOGGER.info("Added edge direction filter iterator of {}", edgeEntityDirFilter);
            }
        }
    } catch (final AccumuloSecurityException | IteratorSettingException | UnsupportedEncodingException e) {
        throw new StoreException(e);
    }
}

Example 37 with Operation

use of uk.gov.gchq.gaffer.operation.Operation in project Gaffer by gchq.

the class GetProxyPropertiesTest method shouldShallowCloneOperation.

@Test
@Override
public void shouldShallowCloneOperation() {
    GetProxyProperties testObject = getTestObject();
    Operation operation = testObject.shallowClone();
    assertEquals(testObject, operation);
    assertFalse(testObject == operation);
}

Example 38 with Operation

use of uk.gov.gchq.gaffer.operation.Operation in project Gaffer by gchq.

the class GraphTest method shouldManipulateViewRemovingBlacklistedEdgeUsingUpdateViewHook.

@Test
public void shouldManipulateViewRemovingBlacklistedEdgeUsingUpdateViewHook() throws OperationException {
    // Given
    operation = new GetElements.Builder().view(new View.Builder().edge(TestGroups.EDGE_5).edge(TestGroups.EDGE).build()).build();
    final UpdateViewHook updateViewHook = new UpdateViewHook.Builder().blackListElementGroups(Collections.singleton(TestGroups.EDGE)).build();
    given(opChain.getOperations()).willReturn(Lists.newArrayList(operation));
    given(opChain.shallowClone()).willReturn(clonedOpChain);
    given(clonedOpChain.getOperations()).willReturn(Lists.newArrayList(operation));
    given(clonedOpChain.flatten()).willReturn(Arrays.asList(operation));
    final Store store = mock(Store.class);
    given(store.getSchema()).willReturn(new Schema());
    given(store.getProperties()).willReturn(new StoreProperties());
    final Graph graph = new Graph.Builder().config(new GraphConfig.Builder().graphId(GRAPH_ID).addHook(updateViewHook).build()).storeProperties(StreamUtil.storeProps(getClass())).store(store).build();
    final ArgumentCaptor<OperationChain> captor = ArgumentCaptor.forClass(OperationChain.class);
    final ArgumentCaptor<Context> contextCaptor1 = ArgumentCaptor.forClass(Context.class);
    given(store.execute(captor.capture(), contextCaptor1.capture())).willReturn(new ArrayList<>());
    // When / Then
    graph.execute(opChain, user);
    final List<Operation> ops = captor.getValue().getOperations();
    JsonAssert.assertEquals(new View.Builder().edge(TestGroups.EDGE_5).build().toCompactJson(), ((GetElements) ops.get(0)).getView().toCompactJson());
}

Example 39 with Operation

use of uk.gov.gchq.gaffer.operation.Operation in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldAddAllOperationsGivenJson.

@Test
public void shouldAddAllOperationsGivenJson() throws IOException {
    // Given
    final byte[] bytes;
    try (final InputStream inputStream = StreamUtil.openStream(getClass(), ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH)) {
        bytes = IOUtils.toByteArray(inputStream);
    }
    final AddOperationsToChain hook = fromJson(bytes);
    Operation discardOutput = new DiscardOutput();
    Operation splitStore = new SplitStoreFromFile();
    Operation validate = new Validate();
    Operation getAdjacentIds = new GetAdjacentIds();
    Operation count = new Count<>();
    Operation countGroups = new CountGroups();
    Operation getElements = new GetElements();
    Operation getAllElements = new GetAllElements();
    Operation limit = new Limit<>();
    final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
    // When
    hook.preExecute(opChain, new Context(new User()));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then(countGroups).then(getElements).then(getAllElements).then(limit).then(validate).then(count).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}

Example 40 with Operation

use of uk.gov.gchq.gaffer.operation.Operation in project Gaffer by gchq.

the class AddOperationsToChainTest method shouldAddAllOperationsWithNoAuthsGivenPath.

@Test
public void shouldAddAllOperationsWithNoAuthsGivenPath() throws IOException {
    // Given
    AddOperationsToChain hook = fromJson(ADD_OPERATIONS_TO_CHAIN_RESOURCE_PATH);
    Operation discardOutput = new DiscardOutput();
    Operation splitStore = new SplitStoreFromFile();
    Operation validate = new Validate();
    Operation getAdjacentIds = new GetAdjacentIds();
    Operation count = new Count<>();
    Operation countGroups = new CountGroups();
    Operation getElements = new GetElements();
    Operation getAllElements = new GetAllElements();
    Operation limit = new Limit<>();
    final OperationChain opChain = new OperationChain.Builder().first(getAdjacentIds).then(getElements).then(getAllElements).build();
    // When
    hook.preExecute(opChain, new Context(new User()));
    // Then
    final OperationChain expectedOpChain = new OperationChain.Builder().first(discardOutput).then(splitStore).then(validate).then(getAdjacentIds).then(count).then(discardOutput).then(countGroups).then(getElements).then(getAllElements).then(limit).then(validate).then(count).build();
    JsonAssert.assertEquals(JSONSerialiser.serialise(expectedOpChain), JSONSerialiser.serialise(opChain));
}