Search in sources :

Example 96 with User

use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.

the class VisibilityIT method shouldAccessMultipleVisibilityGroups_or.

@Test
@TraitRequirement(StoreTrait.VISIBILITY)
public void shouldAccessMultipleVisibilityGroups_or() throws OperationException {
    final Set<Element> elements = new HashSet<>();
    final Entity entity1 = new Entity(TestGroups.ENTITY, "B");
    entity1.putProperty(TestTypes.VISIBILITY, "vis1|vis2");
    elements.add(entity1);
    final AddElements addElements = new AddElements.Builder().input(elements).build();
    graph.execute(addElements, new User());
    final GetElements get = new GetElements.Builder().input(new EntitySeed("B")).build();
    final CloseableIterable<? extends Element> iterable = graph.execute(get, new User(User.UNKNOWN_USER_ID, Sets.newHashSet("vis1")));
    final List<Element> results = Lists.newArrayList(iterable);
    assertThat(results).withFailMessage("Results do not contain all expected entities.").hasSize(1);
    for (final Element e : results) {
        assertThat(e.getProperties()).containsKey(TestTypes.VISIBILITY);
    }
    iterable.close();
}
Also used : AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) Entity(uk.gov.gchq.gaffer.data.element.Entity) User(uk.gov.gchq.gaffer.user.User) Element(uk.gov.gchq.gaffer.data.element.Element) EntitySeed(uk.gov.gchq.gaffer.operation.data.EntitySeed) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) HashSet(java.util.HashSet) Test(org.junit.Test) TraitRequirement(uk.gov.gchq.gaffer.integration.TraitRequirement)

Example 97 with User

use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.

the class AccumuloStore method updateConfiguration.

/**
 * Updates a Hadoop {@link Configuration} with information needed to connect to the Accumulo store. It adds
 * iterators to apply the provided {@link View}. This method will be used by operations that run MapReduce
 * or Spark jobs against the Accumulo store.
 *
 * @param conf         A {@link Configuration} to be updated.
 * @param graphFilters The operation {@link GraphFilters} to be applied.
 * @param user         The {@link User} to be used.
 * @throws StoreException If there is a failure to connect to Accumulo or a problem setting the iterators.
 */
public void updateConfiguration(final Configuration conf, final GraphFilters graphFilters, final User user) throws StoreException {
    try {
        final View view = graphFilters.getView();
        // Table name
        LOGGER.info("Updating configuration with table name of {}", getTableName());
        InputConfigurator.setInputTableName(AccumuloInputFormat.class, conf, getTableName());
        // User
        addUserToConfiguration(conf);
        // Authorizations
        Authorizations authorisations;
        if (null != user && null != user.getDataAuths()) {
            authorisations = new Authorizations(user.getDataAuths().toArray(new String[user.getDataAuths().size()]));
        } else {
            authorisations = new Authorizations();
        }
        InputConfigurator.setScanAuthorizations(AccumuloInputFormat.class, conf, authorisations);
        LOGGER.info("Updating configuration with authorizations of {}", authorisations);
        // Zookeeper
        addZookeeperToConfiguration(conf);
        // Add keypackage, schema and view to conf
        conf.set(ElementInputFormat.KEY_PACKAGE, getProperties().getKeyPackageClass());
        LOGGER.info("Updating configuration with key package of {}", getProperties().getKeyPackageClass());
        conf.set(ElementInputFormat.SCHEMA, new String(getSchema().toCompactJson(), CommonConstants.UTF_8));
        LOGGER.debug("Updating configuration with Schema of {}", getSchema());
        conf.set(ElementInputFormat.VIEW, new String(view.toCompactJson(), CommonConstants.UTF_8));
        LOGGER.debug("Updating configuration with View of {}", view);
        if (view.hasGroups()) {
            // Add the columns to fetch
            final Collection<org.apache.accumulo.core.util.Pair<Text, Text>> columnFamilyColumnQualifierPairs = Stream.concat(view.getEntityGroups().stream(), view.getEdgeGroups().stream()).map(g -> new org.apache.accumulo.core.util.Pair<>(new Text(g), (Text) null)).collect(Collectors.toSet());
            InputConfigurator.fetchColumns(AccumuloInputFormat.class, conf, columnFamilyColumnQualifierPairs);
            LOGGER.info("Updated configuration with column family/qualifiers of {}", StringUtils.join(columnFamilyColumnQualifierPairs, ','));
            // Add iterators that depend on the view
            final IteratorSetting elementPreFilter = getKeyPackage().getIteratorFactory().getElementPreAggregationFilterIteratorSetting(view, this);
            if (null != elementPreFilter) {
                InputConfigurator.addIterator(AccumuloInputFormat.class, conf, elementPreFilter);
                LOGGER.info("Added pre-aggregation filter iterator of {}", elementPreFilter);
            }
            final IteratorSetting elementPostFilter = getKeyPackage().getIteratorFactory().getElementPostAggregationFilterIteratorSetting(view, this);
            if (null != elementPostFilter) {
                InputConfigurator.addIterator(AccumuloInputFormat.class, conf, elementPostFilter);
                LOGGER.info("Added post-aggregation filter iterator of {}", elementPostFilter);
            }
            final IteratorSetting edgeEntityDirFilter = getKeyPackage().getIteratorFactory().getEdgeEntityDirectionFilterIteratorSetting(graphFilters);
            if (null != edgeEntityDirFilter) {
                InputConfigurator.addIterator(AccumuloInputFormat.class, conf, edgeEntityDirFilter);
                LOGGER.info("Added edge direction filter iterator of {}", edgeEntityDirFilter);
            }
        }
    } catch (final AccumuloSecurityException | IteratorSettingException | UnsupportedEncodingException e) {
        throw new StoreException(e);
    }
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) VISIBILITY(uk.gov.gchq.gaffer.store.StoreTrait.VISIBILITY) TableUtils(uk.gov.gchq.gaffer.accumulostore.utils.TableUtils) Text(org.apache.hadoop.io.Text) GenerateSplitPointsFromSampleHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.GenerateSplitPointsFromSampleHandler) Element(uk.gov.gchq.gaffer.data.element.Element) SchemaOptimiser(uk.gov.gchq.gaffer.store.schema.SchemaOptimiser) AddElementsFromHdfsHandler(uk.gov.gchq.gaffer.accumulostore.operation.hdfs.handler.AddElementsFromHdfsHandler) STORE_VALIDATION(uk.gov.gchq.gaffer.store.StoreTrait.STORE_VALIDATION) Configuration(org.apache.hadoop.conf.Configuration) GetElementsBetweenSetsHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.GetElementsBetweenSetsHandler) InputConfigurator(org.apache.accumulo.core.client.mapreduce.lib.impl.InputConfigurator) SampleElementsForSplitPoints(uk.gov.gchq.gaffer.operation.impl.SampleElementsForSplitPoints) ValidationResult(uk.gov.gchq.koryphe.ValidationResult) AccumuloInputFormat(org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat) MutationsRejectedException(org.apache.accumulo.core.client.MutationsRejectedException) Set(java.util.Set) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) GetElementsBetweenSets(uk.gov.gchq.gaffer.accumulostore.operation.impl.GetElementsBetweenSets) Stream(java.util.stream.Stream) AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) StoreTrait(uk.gov.gchq.gaffer.store.StoreTrait) OutputOperationHandler(uk.gov.gchq.gaffer.store.operation.handler.OutputOperationHandler) AddElementsHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.AddElementsHandler) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) AccumuloElementConversionException(uk.gov.gchq.gaffer.accumulostore.key.exception.AccumuloElementConversionException) SplitStore(uk.gov.gchq.gaffer.operation.impl.SplitStore) ImportAccumuloKeyValueFiles(uk.gov.gchq.gaffer.accumulostore.operation.hdfs.operation.ImportAccumuloKeyValueFiles) GetElementsWithinSet(uk.gov.gchq.gaffer.accumulostore.operation.impl.GetElementsWithinSet) Key(org.apache.accumulo.core.data.Key) GetElementsHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.GetElementsHandler) Status(uk.gov.gchq.gaffer.core.exception.Status) SampleDataForSplitPoints(uk.gov.gchq.gaffer.hdfs.operation.SampleDataForSplitPoints) HdfsSplitStoreFromFileHandler(uk.gov.gchq.gaffer.hdfs.operation.handler.HdfsSplitStoreFromFileHandler) ElementInputFormat(uk.gov.gchq.gaffer.accumulostore.inputformat.ElementInputFormat) IteratorSettingException(uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) ColumnVisibility(org.apache.accumulo.core.security.ColumnVisibility) Operation(uk.gov.gchq.gaffer.operation.Operation) SplitStoreFromIterable(uk.gov.gchq.gaffer.operation.impl.SplitStoreFromIterable) Schema(uk.gov.gchq.gaffer.store.schema.Schema) CommonConstants(uk.gov.gchq.gaffer.commonutil.CommonConstants) ChainedIterable(uk.gov.gchq.gaffer.commonutil.iterable.ChainedIterable) OperationHandler(uk.gov.gchq.gaffer.store.operation.handler.OperationHandler) ClientConfiguration(org.apache.accumulo.core.client.ClientConfiguration) GetElementsWithinSetHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.GetElementsWithinSetHandler) SplitStoreFromFile(uk.gov.gchq.gaffer.operation.impl.SplitStoreFromFile) GetAdjacentIdsHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.GetAdjacentIdsHandler) SummariseGroupOverRanges(uk.gov.gchq.gaffer.accumulostore.operation.impl.SummariseGroupOverRanges) LoggerFactory(org.slf4j.LoggerFactory) Mutation(org.apache.accumulo.core.data.Mutation) TRANSFORMATION(uk.gov.gchq.gaffer.store.StoreTrait.TRANSFORMATION) SampleElementsForSplitPointsHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.SampleElementsForSplitPointsHandler) GetAllElementsHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.GetAllElementsHandler) QUERY_AGGREGATION(uk.gov.gchq.gaffer.store.StoreTrait.QUERY_AGGREGATION) CloseableIterable(uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable) POST_TRANSFORMATION_FILTERING(uk.gov.gchq.gaffer.store.StoreTrait.POST_TRANSFORMATION_FILTERING) Value(org.apache.accumulo.core.data.Value) GetElementsInRangesHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.GetElementsInRangesHandler) TypeDefinition(uk.gov.gchq.gaffer.store.schema.TypeDefinition) Pair(uk.gov.gchq.gaffer.commonutil.pair.Pair) Collection(java.util.Collection) SplitStoreHandler(uk.gov.gchq.gaffer.accumulostore.operation.hdfs.handler.SplitStoreHandler) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Store(uk.gov.gchq.gaffer.store.Store) PRE_AGGREGATION_FILTERING(uk.gov.gchq.gaffer.store.StoreTrait.PRE_AGGREGATION_FILTERING) SplitStoreFromIterableHandler(uk.gov.gchq.gaffer.accumulostore.operation.hdfs.handler.SplitStoreFromIterableHandler) List(java.util.List) INGEST_AGGREGATION(uk.gov.gchq.gaffer.store.StoreTrait.INGEST_AGGREGATION) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) Entry(java.util.Map.Entry) UnsupportedEncodingException(java.io.UnsupportedEncodingException) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) SchemaElementDefinition(uk.gov.gchq.gaffer.store.schema.SchemaElementDefinition) GraphFilters(uk.gov.gchq.gaffer.operation.graph.GraphFilters) StoreException(uk.gov.gchq.gaffer.store.StoreException) Serialiser(uk.gov.gchq.gaffer.serialisation.Serialiser) ORDERED(uk.gov.gchq.gaffer.store.StoreTrait.ORDERED) SummariseGroupOverRangesHandler(uk.gov.gchq.gaffer.accumulostore.operation.handler.SummariseGroupOverRangesHandler) User(uk.gov.gchq.gaffer.user.User) Connector(org.apache.accumulo.core.client.Connector) AccumuloKeyPackage(uk.gov.gchq.gaffer.accumulostore.key.AccumuloKeyPackage) GetElementsInRanges(uk.gov.gchq.gaffer.accumulostore.operation.impl.GetElementsInRanges) POST_AGGREGATION_FILTERING(uk.gov.gchq.gaffer.store.StoreTrait.POST_AGGREGATION_FILTERING) ToBytesSerialiser(uk.gov.gchq.gaffer.serialisation.ToBytesSerialiser) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) Max(uk.gov.gchq.koryphe.impl.binaryoperator.Max) EntityId(uk.gov.gchq.gaffer.data.element.id.EntityId) Logger(org.slf4j.Logger) AccumuloStoreConstants(uk.gov.gchq.gaffer.accumulostore.utils.AccumuloStoreConstants) GenerateSplitPointsFromSample(uk.gov.gchq.gaffer.operation.impl.GenerateSplitPointsFromSample) GafferRuntimeException(uk.gov.gchq.gaffer.core.exception.GafferRuntimeException) SampleDataForSplitPointsHandler(uk.gov.gchq.gaffer.accumulostore.operation.hdfs.handler.SampleDataForSplitPointsHandler) Authorizations(org.apache.accumulo.core.security.Authorizations) StoreProperties(uk.gov.gchq.gaffer.store.StoreProperties) MATCHED_VERTEX(uk.gov.gchq.gaffer.store.StoreTrait.MATCHED_VERTEX) GetAdjacentIds(uk.gov.gchq.gaffer.operation.impl.get.GetAdjacentIds) ImportAccumuloKeyValueFilesHandler(uk.gov.gchq.gaffer.accumulostore.operation.hdfs.handler.ImportAccumuloKeyValueFilesHandler) AddElementsFromHdfs(uk.gov.gchq.gaffer.hdfs.operation.AddElementsFromHdfs) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Collections(java.util.Collections) Authorizations(org.apache.accumulo.core.security.Authorizations) UnsupportedEncodingException(java.io.UnsupportedEncodingException) Text(org.apache.hadoop.io.Text) IteratorSettingException(uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) StoreException(uk.gov.gchq.gaffer.store.StoreException) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) Pair(uk.gov.gchq.gaffer.commonutil.pair.Pair)

Example 98 with User

use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.

the class JsonSerialisationConfigIT method shouldSerialiseHyperLogLogPlussesWhenSerialiserModulesConfigured.

@Test
public void shouldSerialiseHyperLogLogPlussesWhenSerialiserModulesConfigured() throws OperationException {
    // Given
    graphFactory.getGraph().execute(new AddElements.Builder().input(new Entity.Builder().vertex("vertex1").group("Cardinality").property("hllp", new HyperLogLogPlus(5, 5)).build()).build(), new User());
    // When
    ResponseEntity<List> elements = post("/graph/operations/execute", new GetAllElements(), List.class);
    Map<String, Object> result = ((List<Map<String, Object>>) elements.getBody()).get(0);
    Map<String, Object> hllp = ((Map<String, Map<String, Map<String, Map<String, Object>>>>) result.get("properties")).get("hllp").get(HyperLogLogPlus.class.getName()).get("hyperLogLogPlus");
    assertNotNull(hllp);
    assertTrue(hllp.containsKey("cardinality"));
}
Also used : AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) Entity(uk.gov.gchq.gaffer.data.element.Entity) ResponseEntity(org.springframework.http.ResponseEntity) User(uk.gov.gchq.gaffer.user.User) HyperLogLogPlus(com.clearspring.analytics.stream.cardinality.HyperLogLogPlus) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) List(java.util.List) Map(java.util.Map) Test(org.junit.Test)

Example 99 with User

use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.

the class OperationControllerTest method shouldCorrectlyChunkIterables.

@Test
public void shouldCorrectlyChunkIterables() throws IOException, OperationException {
    // Given
    when(userFactory.createContext()).thenReturn(new Context(new User()));
    when(store.execute(any(Output.class), any(Context.class))).thenReturn(Arrays.asList(1, 2, 3));
    // When
    ResponseEntity<StreamingResponseBody> response = operationController.executeChunked(new GetAllElements());
    OutputStream output = new ByteArrayOutputStream();
    response.getBody().writeTo(output);
    // Then
    assertEquals("1\r\n2\r\n3\r\n", output.toString());
}
Also used : Context(uk.gov.gchq.gaffer.store.Context) User(uk.gov.gchq.gaffer.user.User) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) Output(uk.gov.gchq.gaffer.operation.io.Output) DiscardOutput(uk.gov.gchq.gaffer.operation.impl.DiscardOutput) ByteArrayOutputStream(java.io.ByteArrayOutputStream) OutputStream(java.io.OutputStream) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Test(org.junit.jupiter.api.Test)

Example 100 with User

use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.

the class FederatedAggregateHandlerTest method shouldAggregateDuplicatesFromDiffStores.

@Test
public void shouldAggregateDuplicatesFromDiffStores() throws Exception {
    FederatedStoreProperties federatedStoreProperties = FederatedStoreProperties.loadStoreProperties(StreamUtil.openStream(currentClass, "predefinedFederatedStore.properties"));
    final Graph fed = new Graph.Builder().config(new GraphConfig("fed")).addSchema(new Schema()).storeProperties(federatedStoreProperties).build();
    final String graphNameA = "a";
    final String graphNameB = "b";
    final Context context = new Context(new User());
    fed.execute(new OperationChain.Builder().first(new AddGraph.Builder().graphId(graphNameA).schema(new Schema.Builder().edge("edge", new SchemaEdgeDefinition.Builder().source("string").destination("string").build()).type("string", String.class).build()).storeProperties(PROPERTIES).build()).then(new AddGraph.Builder().graphId(graphNameB).schema(new Schema.Builder().edge("edge", new SchemaEdgeDefinition.Builder().source("string").destination("string").build()).type("string", String.class).build()).storeProperties(PROPERTIES).build()).build(), context);
    fed.execute(new AddElements.Builder().input(new Edge.Builder().group("edge").source("s1").dest("d1").build()).option(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS, graphNameA).build(), context);
    fed.execute(new AddElements.Builder().input(new Edge.Builder().group("edge").source("s1").dest("d1").build()).option(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS, graphNameB).build(), context);
    final CloseableIterable<? extends Element> getAll = fed.execute(new GetAllElements(), context);
    List<Element> list = new ArrayList<>();
    getAll.forEach(list::add);
    assertThat(list).hasSize(2);
    final Iterable<? extends Element> getAggregate = fed.execute(new OperationChain.Builder().first(new GetAllElements()).then(new Aggregate()).build(), context);
    list.clear();
    getAggregate.forEach(list::add);
    assertThat(list).hasSize(1);
}
Also used : Context(uk.gov.gchq.gaffer.store.Context) User(uk.gov.gchq.gaffer.user.User) FederatedStoreProperties(uk.gov.gchq.gaffer.federatedstore.FederatedStoreProperties) Schema(uk.gov.gchq.gaffer.store.schema.Schema) Element(uk.gov.gchq.gaffer.data.element.Element) ArrayList(java.util.ArrayList) GraphConfig(uk.gov.gchq.gaffer.graph.GraphConfig) AddGraph(uk.gov.gchq.gaffer.federatedstore.operation.AddGraph) Graph(uk.gov.gchq.gaffer.graph.Graph) AddGraph(uk.gov.gchq.gaffer.federatedstore.operation.AddGraph) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) SchemaEdgeDefinition(uk.gov.gchq.gaffer.store.schema.SchemaEdgeDefinition) Aggregate(uk.gov.gchq.gaffer.operation.impl.function.Aggregate) Test(org.junit.jupiter.api.Test)

Aggregations

User (uk.gov.gchq.gaffer.user.User)378 Test (org.junit.jupiter.api.Test)188 Graph (uk.gov.gchq.gaffer.graph.Graph)155 Element (uk.gov.gchq.gaffer.data.element.Element)143 AddElements (uk.gov.gchq.gaffer.operation.impl.add.AddElements)128 GetElements (uk.gov.gchq.gaffer.operation.impl.get.GetElements)110 HashSet (java.util.HashSet)109 View (uk.gov.gchq.gaffer.data.elementdefinition.view.View)104 OperationChain (uk.gov.gchq.gaffer.operation.OperationChain)103 EntitySeed (uk.gov.gchq.gaffer.operation.data.EntitySeed)98 Edge (uk.gov.gchq.gaffer.data.element.Edge)85 Context (uk.gov.gchq.gaffer.store.Context)85 Entity (uk.gov.gchq.gaffer.data.element.Entity)77 Test (org.junit.Test)61 ArrayList (java.util.ArrayList)57 OperationException (uk.gov.gchq.gaffer.operation.OperationException)52 GetAllElements (uk.gov.gchq.gaffer.operation.impl.get.GetAllElements)49 CloseableIterable (uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable)48 ViewElementDefinition (uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition)45 ElementFilter (uk.gov.gchq.gaffer.data.element.function.ElementFilter)43