Search in sources :

Example 1 with RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator

use of uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator in project gaffer-doc by gchq.

the class FederatedStoreWalkThrough method run.

public CloseableIterable<? extends Element> run() throws Exception {
    final Schema schema = new Schema.Builder().json(StreamUtil.openStreams(getClass(), schemaPath)).build();
    final HashMapGraphLibrary library = new HashMapGraphLibrary();
    library.addProperties("mapStore", getMapStoreProperties());
    library.addProperties("accumuloStore", getAccumuloStoreProperties());
    library.addSchema("roadTraffic", schema);
    // [creating a federatedstore] create a store that federates to a MapStore and AccumuloStore
    // ---------------------------------------------------------
    final Graph federatedGraph = new Graph.Builder().config(new GraphConfig.Builder().graphId(getClass().getSimpleName()).library(library).build()).storeProperties(getFederatedStoreProperties()).build();
    // ---------------------------------------------------------
    // [federatedstore properties]
    // ---------------------------------------------------------
    final FederatedStoreProperties exampleFederatedProperty = new FederatedStoreProperties();
    exampleFederatedProperty.setCacheProperties(HashMapCacheService.class.getName());
    // ---------------------------------------------------------
    printJson("FED_PROP", getFederatedStoreProperties());
    final User user = new User("user01");
    final AddGraph addMapGraph = new AddGraph.Builder().graphId("mapGraph").schema(new Schema.Builder().json(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalitiesForFederatedStore/schema/entities.json")).json(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalitiesForFederatedStore/schema/types.json")).build()).storeProperties(getMapStoreProperties()).isPublic(true).build();
    federatedGraph.execute(addMapGraph, user);
    final AddGraph addAccumuloGraph = new AddGraph.Builder().graphId("accumuloGraph").schema(new Schema.Builder().json(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalitiesForFederatedStore/schema/edges.json")).json(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalitiesForFederatedStore/schema/types.json")).build()).storeProperties(getAccumuloStoreProperties()).isPublic(true).build();
    federatedGraph.execute(addAccumuloGraph, user);
    // [add another graph] add a graph to the federated store.
    // ---------------------------------------------------------
    AddGraph addAnotherGraph = new AddGraph.Builder().graphId("AnotherGraph").schema(schema).storeProperties(getMapStoreProperties()).build();
    federatedGraph.execute(addAnotherGraph, user);
    // ---------------------------------------------------------
    improveReadabilityOfJson(addAnotherGraph);
    addAccumuloGraph.setGraphAuths(null);
    printJson("ADD_GRAPH", addAnotherGraph);
    // [remove graph] remove a graph from the federated store.
    // ---------------------------------------------------------
    RemoveGraph removeGraph = new RemoveGraph.Builder().graphId("AnotherGraph").build();
    federatedGraph.execute(removeGraph, user);
    // ---------------------------------------------------------
    improveReadabilityOfJson(removeGraph);
    printJson("REMOVE_GRAPH", removeGraph);
    // [get all graph ids] Get a list of all the graphId within the FederatedStore.
    // ---------------------------------------------------------
    final GetAllGraphIds getAllGraphIDs = new GetAllGraphIds();
    Iterable<? extends String> graphIds = federatedGraph.execute(getAllGraphIDs, user);
    // ---------------------------------------------------------
    improveReadabilityOfJson(getAllGraphIDs);
    printJson("GET_ALL_GRAPH_IDS", getAllGraphIDs);
    print("GRAPH_IDS", graphIds.toString());
    // [add elements] Create a data generator and add the edges to the federated graphs using an operation chain consisting of:
    // generateElements - generating edges from the data (note these are directed edges)
    // addElements - add the edges to the graph
    // ---------------------------------------------------------
    final OperationChain<Void> addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator()).input(IOUtils.readLines(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalities/data.txt"))).build()).then(new AddElements()).build();
    federatedGraph.execute(addOpChain, user);
    // ---------------------------------------------------------
    printJsonAndPython("ADD_ELEMENTS", addOpChain);
    // [get elements]
    // ---------------------------------------------------------
    final OperationChain<CloseableIterable<? extends Element>> getOpChain = new OperationChain.Builder().first(new GetElements.Builder().input(new EntitySeed("10")).build()).build();
    CloseableIterable<? extends Element> elements = federatedGraph.execute(getOpChain, user);
    for (final Element element : elements) {
        print("ELEMENTS", element.toString());
    }
    printJsonAndPython("GET_ELEMENTS", getOpChain);
    // [get elements from accumulo graph]
    // ---------------------------------------------------------
    final OperationChain<CloseableIterable<? extends Element>> getOpChainOnAccumuloGraph = new OperationChain.Builder().first(new GetElements.Builder().input(new EntitySeed("10")).option(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS, "accumuloGraph").build()).build();
    CloseableIterable<? extends Element> elementsFromAccumuloGraph = federatedGraph.execute(getOpChainOnAccumuloGraph, user);
    for (final Element element : elementsFromAccumuloGraph) {
        print("ELEMENTS_FROM_ACCUMULO_GRAPH", element.toString());
    }
    // [select graphs for operations]
    // ---------------------------------------------------------
    GetAllElements selectGraphsForOperations = new Builder().option(FederatedStoreConstants.KEY_OPERATION_OPTIONS_GRAPH_IDS, "graphId1, graphId2").build();
    // ---------------------------------------------------------
    printJsonAndPython("SELECT_GRAPHS_FOR_OPERATIONS", selectGraphsForOperations);
    // [do not skip failed execution]
    // ---------------------------------------------------------
    GetAllElements doNotSkipFailedExecution = new Builder().option(FederatedStoreConstants.KEY_SKIP_FAILED_FEDERATED_STORE_EXECUTE, "false").build();
    // ---------------------------------------------------------
    printJsonAndPython("DO_NOT_SKIP_FAILED_EXECUTION", doNotSkipFailedExecution);
    // [add public graph] add a graph to the federated store.
    // ---------------------------------------------------------
    AddGraph publicGraph = new AddGraph.Builder().graphId("publicGraph").parentSchemaIds(Lists.newArrayList("roadTraffic")).parentPropertiesId("mapStore").isPublic(// <-- public access
    true).graphAuths(// <-- used but irrelevant as graph has public access
    "Auth1").build();
    federatedGraph.execute(addAnotherGraph, user);
    // ---------------------------------------------------------
    improveReadabilityOfJson(publicGraph);
    printJson("ADD_PUBLIC_GRAPH", publicGraph);
    // [add private graph] add a graph to the federated store.
    // ---------------------------------------------------------
    AddGraph privateGraph = new AddGraph.Builder().graphId("privateGraph").parentSchemaIds(Lists.newArrayList("roadTraffic")).parentPropertiesId("mapStore").build();
    federatedGraph.execute(addAnotherGraph, user);
    // ---------------------------------------------------------
    improveReadabilityOfJson(privateGraph);
    printJson("ADD_PRIVATE_GRAPH", privateGraph);
    // [add secure graph] add a graph to the federated store.
    // ---------------------------------------------------------
    AddGraph addSecureGraph = new AddGraph.Builder().graphId("SecureGraph").parentSchemaIds(Lists.newArrayList("roadTraffic")).parentPropertiesId("mapStore").graphAuths("Auth1", "Auth2", "Auth3").build();
    federatedGraph.execute(addSecureGraph, user);
    // ---------------------------------------------------------
    improveReadabilityOfJson(addSecureGraph);
    printJson("ADD_SECURE_GRAPH", addSecureGraph);
    // [disallow public access]
    // ---------------------------------------------------------
    FederatedStoreProperties disallowPublicProperties = new FederatedStoreProperties();
    disallowPublicProperties.setGraphsCanHavePublicAccess(false);
    // ---------------------------------------------------------
    printJson("DISALLOW_PUBLIC_ACCESS", disallowPublicProperties.getProperties());
    // [limit custom properties]
    // ---------------------------------------------------------
    FederatedStoreProperties limitCustomProperties = new FederatedStoreProperties();
    limitCustomProperties.setCustomPropertyAuths("Auth1, Auth2, Auth3");
    // ---------------------------------------------------------
    printJson("LIMIT_CUSTOM_PROPERTIES", limitCustomProperties.getProperties());
    return elements;
}
Also used : AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) User(uk.gov.gchq.gaffer.user.User) FederatedStoreProperties(uk.gov.gchq.gaffer.federatedstore.FederatedStoreProperties) Schema(uk.gov.gchq.gaffer.store.schema.Schema) Builder(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements.Builder) Element(uk.gov.gchq.gaffer.data.element.Element) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) HashMapCacheService(uk.gov.gchq.gaffer.cache.impl.HashMapCacheService) GetAllGraphIds(uk.gov.gchq.gaffer.federatedstore.operation.GetAllGraphIds) AddGraph(uk.gov.gchq.gaffer.federatedstore.operation.AddGraph) RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator(uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) RemoveGraph(uk.gov.gchq.gaffer.federatedstore.operation.RemoveGraph) HashMapGraphLibrary(uk.gov.gchq.gaffer.store.library.HashMapGraphLibrary) CloseableIterable(uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable) Graph(uk.gov.gchq.gaffer.graph.Graph) RemoveGraph(uk.gov.gchq.gaffer.federatedstore.operation.RemoveGraph) AddGraph(uk.gov.gchq.gaffer.federatedstore.operation.AddGraph) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) EntitySeed(uk.gov.gchq.gaffer.operation.data.EntitySeed)

Example 2 with RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator

use of uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator in project gaffer-doc by gchq.

the class Cardinalities method run.

public CloseableIterable<? extends Element> run() throws OperationException, IOException {
    // [graph] create a graph using our schema and store properties
    // ---------------------------------------------------------
    final Graph graph = new Graph.Builder().config(getDefaultGraphConfig()).addSchemas(StreamUtil.openStreams(getClass(), schemaPath)).storeProperties(getDefaultStoreProperties()).build();
    // ---------------------------------------------------------
    // [user] Create a user
    // ---------------------------------------------------------
    final User user = new User("user01");
    // ---------------------------------------------------------
    // [add] Create a data generator and add the edges to the graph using an operation chain consisting of:
    // generateElements - generating edges from the data (note these are directed edges)
    // addElements - add the edges to the graph
    // ---------------------------------------------------------
    final OperationChain<Void> addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator()).input(IOUtils.readLines(StreamUtil.openStream(getClass(), dataPath))).build()).then(new AddElements()).build();
    graph.execute(addOpChain, user);
    // ---------------------------------------------------------
    print("The elements have been added.");
    // [get] Get all edges
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> edges = graph.execute(new GetAllElements(), user);
    // ---------------------------------------------------------
    print("\nAll edges:");
    for (final Element edge : edges) {
        print("GET_ALL_EDGES_RESULT", edge.toString());
    }
    // [get all cardinalities] Get all cardinalities
    // ---------------------------------------------------------
    final GetAllElements getAllCardinalities = new GetAllElements.Builder().view(new View.Builder().entity("Cardinality").build()).build();
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> allCardinalities = graph.execute(getAllCardinalities, user);
    print("\nAll cardinalities");
    for (final Element cardinality : allCardinalities) {
        final String edgeGroup = cardinality.getProperty("edgeGroup").toString();
        print("ALL_CARDINALITIES_RESULT", "Vertex " + ((Entity) cardinality).getVertex() + " " + edgeGroup + ": " + ((HyperLogLogPlus) cardinality.getProperty("hllp")).cardinality());
    }
    // [get all summarised cardinalities] Get all summarised cardinalities over all edges
    // ---------------------------------------------------------
    final GetAllElements getAllSummarisedCardinalities = new GetAllElements.Builder().view(new View.Builder().entity("Cardinality", new ViewElementDefinition.Builder().groupBy().build()).build()).build();
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> allSummarisedCardinalities = graph.execute(getAllSummarisedCardinalities, user);
    print("\nAll summarised cardinalities");
    for (final Element cardinality : allSummarisedCardinalities) {
        final String edgeGroup = cardinality.getProperty("edgeGroup").toString();
        print("ALL_SUMMARISED_CARDINALITIES_RESULT", "Vertex " + ((Entity) cardinality).getVertex() + " " + edgeGroup + ": " + ((HyperLogLogPlus) cardinality.getProperty("hllp")).cardinality());
    }
    // [get roaduse edge cardinality 10] Get the cardinality value at vertex 10 for RoadUse edges
    // ---------------------------------------------------------
    final GetElements getCardinalities = new GetElements.Builder().input(new EntitySeed("10")).view(new View.Builder().entity("Cardinality", new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select("edgeGroup").execute(new IsEqual(CollectionUtil.treeSet("RoadUse"))).build()).build()).build()).build();
    // ---------------------------------------------------------
    final Element roadUse10Cardinality;
    try (final CloseableIterable<? extends Element> elements = graph.execute(getCardinalities, user)) {
        roadUse10Cardinality = elements.iterator().next();
    }
    print("\nRoadUse edge cardinality at vertex 10:");
    final String edgeGroup = (roadUse10Cardinality.getProperty("edgeGroup")).toString();
    print("CARDINALITY_OF_10_RESULT", "Vertex " + ((Entity) roadUse10Cardinality).getVertex() + " " + edgeGroup + ": " + ((HyperLogLogPlus) roadUse10Cardinality.getProperty("hllp")).cardinality());
    // [get 2 hops with a cardinality filter] 2 hops with a cardinality filter
    // ---------------------------------------------------------
    final OperationChain<CloseableIterable<? extends Element>> twoHopsWithCardinalityFilter = new OperationChain.Builder().first(new GetElements.Builder().input(new EntitySeed("M5")).view(new View.Builder().edge("RoadHasJunction").build()).build()).then(new GetElements.Builder().view(new View.Builder().entity("Cardinality", new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select("edgeGroup").execute(new IsEqual(CollectionUtil.treeSet("RoadUse"))).build()).groupBy().postAggregationFilter(new ElementFilter.Builder().select("hllp").execute(new HyperLogLogPlusIsLessThan(5)).build()).build()).build()).build()).then(new GetElements.Builder().view(new View.Builder().edge("RoadUse").build()).build()).build();
    // ---------------------------------------------------------
    final CloseableIterable<? extends Element> twoHopsWithCardinalityFilterResult = graph.execute(twoHopsWithCardinalityFilter, user);
    print("\n2 hops with cardinality filter result:");
    for (final Element element : twoHopsWithCardinalityFilterResult) {
        print("2_HOP_RESULT", element.toString());
    }
    return allSummarisedCardinalities;
}
Also used : AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) Entity(uk.gov.gchq.gaffer.data.element.Entity) User(uk.gov.gchq.gaffer.user.User) Element(uk.gov.gchq.gaffer.data.element.Element) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) ViewElementDefinition(uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition) RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator(uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator) GetAllElements(uk.gov.gchq.gaffer.operation.impl.get.GetAllElements) HyperLogLogPlusIsLessThan(uk.gov.gchq.gaffer.sketches.clearspring.cardinality.predicate.HyperLogLogPlusIsLessThan) CloseableIterable(uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) IsEqual(uk.gov.gchq.koryphe.impl.predicate.IsEqual) Graph(uk.gov.gchq.gaffer.graph.Graph) HyperLogLogPlus(com.clearspring.analytics.stream.cardinality.HyperLogLogPlus) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) ElementFilter(uk.gov.gchq.gaffer.data.element.function.ElementFilter) EntitySeed(uk.gov.gchq.gaffer.operation.data.EntitySeed)

Example 3 with RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator

use of uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator in project gaffer-doc by gchq.

the class Subgraphs method run.

@Override
public Iterable<? extends Element> run() throws OperationException, IOException {
    // / [graph] create a graph using our schema and store properties
    // ---------------------------------------------------------
    final Graph graph = new Graph.Builder().config(getDefaultGraphConfig()).addSchemas(StreamUtil.openStreams(getClass(), schemaPath)).storeProperties(getDefaultStoreProperties()).build();
    // ---------------------------------------------------------
    // [user] Create a user
    // ---------------------------------------------------------
    final User user = new User("user01");
    // ---------------------------------------------------------
    // [add] Create a data generator and add the edges to the graph using an operation chain consisting of:
    // generateElements - generating edges from the data (note these are directed edges)
    // addElements - add the edges to the graph
    // ---------------------------------------------------------
    final OperationChain<Void> addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator()).input(IOUtils.readLines(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalities/data.txt"))).build()).then(new AddElements()).build();
    graph.execute(addOpChain, user);
    // ---------------------------------------------------------
    print("The elements have been added.");
    // [get] Create a sub graph
    // Start getting related edges with the given seeds.
    // Then update the export with the results
    // Between each hop we need to extract the destination vertices of the
    // previous edges and convert them to EntitySeeds.
    // Finally finish off by returning all the edges in the export.
    // ---------------------------------------------------------
    final OperationChain<Iterable<?>> opChain = new OperationChain.Builder().first(new GetElements.Builder().input(new EntitySeed("M5")).inOutType(IncludeIncomingOutgoingType.OUTGOING).view(new View.Builder().edge("RoadHasJunction").build()).build()).then(new ExportToSet<>()).then(new ToVertices.Builder().edgeVertices(ToVertices.EdgeVertices.DESTINATION).build()).then(new ToEntitySeeds()).then(new GetElements.Builder().inOutType(IncludeIncomingOutgoingType.OUTGOING).view(new View.Builder().edge("RoadUse", new ViewElementDefinition.Builder().preAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(1L)).build()).build()).build()).build()).then(new ExportToSet<>()).then(new DiscardOutput()).then(new GetSetExport()).build();
    // ---------------------------------------------------------
    final Iterable<? extends Element> subGraph = (Iterable<? extends Element>) graph.execute(opChain, user);
    print("\nSub graph:");
    for (final Element edge : subGraph) {
        print("SUB_GRAPH", edge.toString());
    }
    return subGraph;
}
Also used : AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) User(uk.gov.gchq.gaffer.user.User) ExportToSet(uk.gov.gchq.gaffer.operation.impl.export.set.ExportToSet) ToEntitySeeds(uk.gov.gchq.gaffer.operation.impl.output.ToEntitySeeds) Element(uk.gov.gchq.gaffer.data.element.Element) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) ViewElementDefinition(uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition) GetSetExport(uk.gov.gchq.gaffer.operation.impl.export.set.GetSetExport) RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator(uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator) ToVertices(uk.gov.gchq.gaffer.operation.impl.output.ToVertices) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) Graph(uk.gov.gchq.gaffer.graph.Graph) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) EntitySeed(uk.gov.gchq.gaffer.operation.data.EntitySeed) DiscardOutput(uk.gov.gchq.gaffer.operation.impl.DiscardOutput) IsMoreThan(uk.gov.gchq.koryphe.impl.predicate.IsMoreThan)

Example 4 with RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator

use of uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator in project gaffer-doc by gchq.

the class Views method run.

public CloseableIterable<? extends Element> run() throws OperationException, IOException {
    // [graph] create a graph using our schema and store properties
    // ---------------------------------------------------------
    final Graph graph = new Graph.Builder().config(getDefaultGraphConfig()).addSchemas(StreamUtil.openStreams(getClass(), schemaPath)).storeProperties(getDefaultStoreProperties()).build();
    // ---------------------------------------------------------
    // [user] Create a user
    // ---------------------------------------------------------
    final User user = new User("user01");
    // ---------------------------------------------------------
    // [add] Create a data generator and add the edges to the graph using an operation chain consisting of:
    // generateElements - generating edges from the data (note these are directed edges)
    // addElements - add the edges to the graph
    // ---------------------------------------------------------
    final OperationChain<Void> addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator()).input(IOUtils.readLines(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalities/data.txt"))).build()).then(new AddElements()).build();
    graph.execute(addOpChain, user);
    // ---------------------------------------------------------
    print("The elements have been added.");
    // [view with groups]
    // ---------------------------------------------------------
    final View viewWithGroups = new View.Builder().edge("RoadUse").edge("RoadHasJunction").entity("Cardinality").build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_GROUPS", viewWithGroups);
    // [view with filters]
    // ---------------------------------------------------------
    final View viewWithFilters = new View.Builder().edge("RoadUse", new ViewElementDefinition.Builder().postAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(2L)).build()).build()).entity("Cardinality", new ViewElementDefinition.Builder().postAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(5L)).select("hllp").execute(new HyperLogLogPlusIsLessThan(10L)).build()).build()).build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_FILTERS", viewWithFilters);
    // [view with removed properties]
    // ---------------------------------------------------------
    final View viewWithRemovedProperties = new View.Builder().edge("RoadUse", new ViewElementDefinition.Builder().properties("count").build()).entity("Cardinality", new ViewElementDefinition.Builder().excludeProperties("hllp", "edgeGroups").build()).build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_REMOVED_PROPERTIES", viewWithRemovedProperties);
    // [view with global filter] run query with a global filter to return only elements with a count more than 2
    // ---------------------------------------------------------
    final View viewWithGlobalFilter = new View.Builder().globalElements(new GlobalViewElementDefinition.Builder().postAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(2L)).build()).build()).build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_GLOBAL_FILTER", viewWithGlobalFilter);
    // [view with global and specific filters]
    // ---------------------------------------------------------
    final View globalAndSpecificView = new View.Builder().globalElements(new GlobalViewElementDefinition.Builder().postAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(0L)).build()).build()).edge("RoadUse", new ViewElementDefinition.Builder().postAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(2L)).build()).build()).entity("Cardinality").build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_GLOBAL_AND_SPECIFIC_FILTERS", globalAndSpecificView);
    // [view with global and specific filters expanded]
    // ---------------------------------------------------------
    final View globalAndSpecificViewExpanded = new View.Builder().edge("RoadUse", new ViewElementDefinition.Builder().postAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(2L)).select("count").execute(new IsMoreThan(0L)).build()).build()).entity("Cardinality", new ViewElementDefinition.Builder().postAggregationFilter(new ElementFilter.Builder().select("count").execute(new IsMoreThan(0L)).build()).build()).build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_GLOBAL_AND_SPECIFIC_FILTERS_EXPANDED", globalAndSpecificViewExpanded);
    // [view with global removed properties]
    // ---------------------------------------------------------
    final View viewWithGlobalRemovedProperties = new View.Builder().globalElements(new GlobalViewElementDefinition.Builder().properties("count").build()).build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_GLOBAL_REMOVED_PROPERTIES", viewWithGlobalRemovedProperties);
    // [view with global and specific removed properties]
    // ---------------------------------------------------------
    final View viewWithGlobalAndSpecificRemovedProperties = new View.Builder().globalElements(new GlobalViewElementDefinition.Builder().properties("count").build()).edge("RoadUse").entity("Cardinality", new ViewElementDefinition.Builder().properties("hllp").build()).build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_GLOBAL_AND_SPECIFIC_REMOVED_PROPERTIES", viewWithGlobalAndSpecificRemovedProperties);
    // [view with global and specific removed properties expanded]
    // ---------------------------------------------------------
    final View viewWithGlobalAndSpecificRemovedPropertiesExpanded = new View.Builder().edge("RoadUse", new ViewElementDefinition.Builder().properties("count").build()).entity("Cardinality", new ViewElementDefinition.Builder().properties("hllp").build()).build();
    // ---------------------------------------------------------
    printJsonAndPythonWithClass("VIEW_WITH_GLOBAL_AND_SPECIFIC_REMOVED_PROPERTIES_EXPANDED", viewWithGlobalAndSpecificRemovedPropertiesExpanded);
    return null;
}
Also used : AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) User(uk.gov.gchq.gaffer.user.User) GlobalViewElementDefinition(uk.gov.gchq.gaffer.data.elementdefinition.view.GlobalViewElementDefinition) ViewElementDefinition(uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) Graph(uk.gov.gchq.gaffer.graph.Graph) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator(uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator) ElementFilter(uk.gov.gchq.gaffer.data.element.function.ElementFilter) HyperLogLogPlusIsLessThan(uk.gov.gchq.gaffer.sketches.clearspring.cardinality.predicate.HyperLogLogPlusIsLessThan) IsMoreThan(uk.gov.gchq.koryphe.impl.predicate.IsMoreThan)

Example 5 with RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator

use of uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator in project gaffer-doc by gchq.

the class Jobs method run.

public CloseableIterable<? extends Element> run() throws OperationException, IOException {
    // / [graph] create a graph using our schema and store properties
    // ---------------------------------------------------------
    final Graph graph = new Graph.Builder().config(getDefaultGraphConfig()).addSchemas(StreamUtil.openStreams(getClass(), schemaPath)).storeProperties(getDefaultStoreProperties()).build();
    // ---------------------------------------------------------
    // [user] Create a user
    // ---------------------------------------------------------
    final User user = new User("user01");
    // ---------------------------------------------------------
    // [add] Create a data generator and add the edges to the graph using an operation chain consisting of:
    // generateElements - generating edges from the data (note these are directed edges)
    // addElements - add the edges to the graph
    // ---------------------------------------------------------
    final OperationChain<Void> addOpChain = new OperationChain.Builder().first(new GenerateElements.Builder<String>().generator(new RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator()).input(IOUtils.readLines(StreamUtil.openStream(getClass(), "RoadAndRoadUseWithTimesAndCardinalities/data.txt"))).build()).then(new AddElements()).build();
    graph.execute(addOpChain, user);
    // ---------------------------------------------------------
    // [job] create an operation chain to be executed as a job
    // ---------------------------------------------------------
    final OperationChain<CloseableIterable<? extends Element>> job = new OperationChain.Builder().first(new GetElements.Builder().input(new EntitySeed("10")).view(new View.Builder().edge("RoadUse").build()).build()).build();
    // ---------------------------------------------------------
    // [execute job] execute the job
    // ---------------------------------------------------------
    final JobDetail initialJobDetail = graph.executeJob(job, user);
    final String jobId = initialJobDetail.getJobId();
    // ---------------------------------------------------------
    print("JOB_DETAIL_START", initialJobDetail.toString());
    waitUntilJobHashFinished(user, graph, initialJobDetail);
    // [job details] Get the job details
    // ---------------------------------------------------------
    final JobDetail jobDetail = graph.execute(new GetJobDetails.Builder().jobId(jobId).build(), user);
    // ---------------------------------------------------------
    print("JOB_DETAIL_FINISH", jobDetail.toString());
    // [all job details] Get all job details
    // ---------------------------------------------------------
    final CloseableIterable<JobDetail> jobDetails = graph.execute(new GetAllJobDetails(), user);
    // ---------------------------------------------------------
    for (final JobDetail detail : jobDetails) {
        print("ALL_JOB_DETAILS", detail.toString());
    }
    // [get job results] Get the job results
    // ---------------------------------------------------------
    final CloseableIterable<?> jobResults = graph.execute(new GetJobResults.Builder().jobId(jobId).build(), user);
    // ---------------------------------------------------------
    for (final Object result : jobResults) {
        print("JOB_RESULTS", result.toString());
    }
    return (CloseableIterable) jobResults;
}
Also used : AddElements(uk.gov.gchq.gaffer.operation.impl.add.AddElements) User(uk.gov.gchq.gaffer.user.User) GetJobResults(uk.gov.gchq.gaffer.operation.impl.job.GetJobResults) Element(uk.gov.gchq.gaffer.data.element.Element) GetElements(uk.gov.gchq.gaffer.operation.impl.get.GetElements) JobDetail(uk.gov.gchq.gaffer.jobtracker.JobDetail) GetAllJobDetails(uk.gov.gchq.gaffer.operation.impl.job.GetAllJobDetails) RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator(uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator) GetJobDetails(uk.gov.gchq.gaffer.operation.impl.job.GetJobDetails) CloseableIterable(uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable) Graph(uk.gov.gchq.gaffer.graph.Graph) OperationChain(uk.gov.gchq.gaffer.operation.OperationChain) EntitySeed(uk.gov.gchq.gaffer.operation.data.EntitySeed)

Aggregations

RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator (uk.gov.gchq.gaffer.doc.user.generator.RoadAndRoadUseWithTimesAndCardinalitiesElementGenerator)7 Graph (uk.gov.gchq.gaffer.graph.Graph)7 OperationChain (uk.gov.gchq.gaffer.operation.OperationChain)7 AddElements (uk.gov.gchq.gaffer.operation.impl.add.AddElements)7 User (uk.gov.gchq.gaffer.user.User)7 EntitySeed (uk.gov.gchq.gaffer.operation.data.EntitySeed)6 GetElements (uk.gov.gchq.gaffer.operation.impl.get.GetElements)6 Element (uk.gov.gchq.gaffer.data.element.Element)5 View (uk.gov.gchq.gaffer.data.elementdefinition.view.View)5 CloseableIterable (uk.gov.gchq.gaffer.commonutil.iterable.CloseableIterable)4 ViewElementDefinition (uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition)3 ElementFilter (uk.gov.gchq.gaffer.data.element.function.ElementFilter)2 GetAllElements (uk.gov.gchq.gaffer.operation.impl.get.GetAllElements)2 HyperLogLogPlusIsLessThan (uk.gov.gchq.gaffer.sketches.clearspring.cardinality.predicate.HyperLogLogPlusIsLessThan)2 IsMoreThan (uk.gov.gchq.koryphe.impl.predicate.IsMoreThan)2 HyperLogLogPlus (com.clearspring.analytics.stream.cardinality.HyperLogLogPlus)1 HashMapCacheService (uk.gov.gchq.gaffer.cache.impl.HashMapCacheService)1 Entity (uk.gov.gchq.gaffer.data.element.Entity)1 EntityId (uk.gov.gchq.gaffer.data.element.id.EntityId)1 GlobalViewElementDefinition (uk.gov.gchq.gaffer.data.elementdefinition.view.GlobalViewElementDefinition)1