use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.
the class JcsJobTrackerIT method shouldAddJobIdToJobTrackerWhenExecute.
@Test
public void shouldAddJobIdToJobTrackerWhenExecute() throws OperationException, IOException, InterruptedException {
// Given
final OperationChain<JobDetail> opChain = new OperationChain.Builder().first(new GetAllEdges()).then(new GetJobDetails()).build();
final User user = new User("user01");
// When
final JobDetail jobDetails = graph.execute(opChain, user);
final String jobId = jobDetails.getJobId();
final JobDetail expectedJobDetail = new JobDetail(jobId, user.getUserId(), opChain, JobStatus.RUNNING, null);
expectedJobDetail.setStartTime(jobDetails.getStartTime());
expectedJobDetail.setEndTime(jobDetails.getEndTime());
// Then
assertEquals(expectedJobDetail, jobDetails);
}
use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.
the class JcsJobTrackerIT method shouldAddJobIdToJobTrackerWhenExecuteJob.
@Test
public void shouldAddJobIdToJobTrackerWhenExecuteJob() throws OperationException, IOException, InterruptedException {
// Given
final OperationChain<CloseableIterable<Edge>> opChain = new OperationChain<>(new GetAllEdges());
final User user = new User("user01");
// When
JobDetail jobDetails = graph.executeJob(opChain, user);
final String jobId = jobDetails.getJobId();
final JobDetail expectedJobDetail = new JobDetail(jobId, user.getUserId(), opChain, JobStatus.FINISHED, null);
expectedJobDetail.setStartTime(jobDetails.getStartTime());
int count = 0;
while (JobStatus.RUNNING == jobDetails.getStatus() && ++count < 20) {
Thread.sleep(100);
jobDetails = graph.execute(new GetJobDetails.Builder().jobId(jobId).build(), user);
}
expectedJobDetail.setEndTime(jobDetails.getEndTime());
// Then
assertEquals(expectedJobDetail, jobDetails);
}
use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.
the class GetJavaRDDOfElementsExample method getJavaRddOfElements.
public void getJavaRddOfElements(final JavaSparkContext sc, final Graph graph) throws OperationException {
ROOT_LOGGER.setLevel(Level.INFO);
// Avoid using getMethodNameAsSentence as it messes up the formatting of the "RDD" part
log("#### get Java RDD of elements\n");
printGraph();
ROOT_LOGGER.setLevel(Level.OFF);
final GetJavaRDDOfElements<ElementSeed> operation = new GetJavaRDDOfElements.Builder<>().addSeed(new EdgeSeed(1, 2, true)).addSeed(new EdgeSeed(2, 3, true)).javaSparkContext(sc).build();
final JavaRDD<Element> rdd = graph.execute(operation, new User("user01"));
final List<Element> elements = rdd.collect();
ROOT_LOGGER.setLevel(Level.INFO);
printJava("GetJavaRDDOfElements<ElementSeed> operation = new GetJavaRDDOfElements.Builder<>()\n" + " .addSeed(new EdgeSeed(1, 2, true))\n" + " .addSeed(new EdgeSeed(2, 3, true))\n" + " .javaSparkContext(sc)\n" + " .build();\n" + "JavaRDD<Element> rdd = graph.execute(operation, new User(\"user01\"));\n" + "List<Element> elements = rdd.collect();");
log("The results are:");
log("```");
for (final Element e : elements) {
log(e.toString());
}
log("```");
ROOT_LOGGER.setLevel(Level.OFF);
}
use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.
the class LoadAndQuery1 method run.
public CloseableIterable<Edge> run() throws OperationException {
// [user] Create a user
// ---------------------------------------------------------
final User user = new User("user01");
// ---------------------------------------------------------
// [generate] Create some edges from the data file using our data generator class
// ---------------------------------------------------------
final List<Element> elements = new ArrayList<>();
final DataGenerator1 dataGenerator = new DataGenerator1();
for (final String s : DataUtils.loadData(getData())) {
elements.add(dataGenerator.getElement(s));
}
// ---------------------------------------------------------
log("Elements generated from the data file.");
for (final Element element : elements) {
log("GENERATED_EDGES", element.toString());
}
log("");
// [graph] Create a graph using our schema and store properties
// ---------------------------------------------------------
final Graph graph = new Graph.Builder().addSchemas(getSchemas()).storeProperties(getStoreProperties()).build();
// ---------------------------------------------------------
// [add] Add the edges to the graph
// ---------------------------------------------------------
final AddElements addElements = new AddElements.Builder().elements(elements).build();
graph.execute(addElements, user);
// ---------------------------------------------------------
log("The elements have been added.\n");
// [get] Get all the edges that contain the vertex "1"
// ---------------------------------------------------------
final GetEdges<EntitySeed> query = new GetEdges.Builder<EntitySeed>().addSeed(new EntitySeed("1")).build();
final CloseableIterable<Edge> results = graph.execute(query, user);
// ---------------------------------------------------------
log("All edges containing the vertex 1. The counts have been aggregated.");
for (final Element e : results) {
log("GET_RELATED_EDGES_RESULT", e.toString());
}
return results;
}
use of uk.gov.gchq.gaffer.user.User in project Gaffer by gchq.
the class GetJavaRDDOfElementsHandlerTest method checkGetCorrectElementsInRDDForEdgeSeed.
@Test
public void checkGetCorrectElementsInRDDForEdgeSeed() throws OperationException, IOException {
final Graph graph1 = new Graph.Builder().addSchema(getClass().getResourceAsStream("/schema/dataSchema.json")).addSchema(getClass().getResourceAsStream("/schema/dataTypes.json")).addSchema(getClass().getResourceAsStream("/schema/storeTypes.json")).storeProperties(getClass().getResourceAsStream("/store.properties")).build();
final List<Element> elements = new ArrayList<>();
for (int i = 0; i < 10; i++) {
final Entity entity = new Entity(ENTITY_GROUP);
entity.setVertex("" + i);
final Edge edge1 = new Edge(EDGE_GROUP);
edge1.setSource("" + i);
edge1.setDestination("B");
edge1.setDirected(false);
edge1.putProperty("count", 2);
final Edge edge2 = new Edge(EDGE_GROUP);
edge2.setSource("" + i);
edge2.setDestination("C");
edge2.setDirected(false);
edge2.putProperty("count", 4);
elements.add(edge1);
elements.add(edge2);
elements.add(entity);
}
final User user = new User();
graph1.execute(new AddElements(elements), user);
final SparkConf sparkConf = new SparkConf().setMaster("local").setAppName("testCheckGetCorrectElementsInJavaRDDForEdgeSeed").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer").set("spark.kryo.registrator", "uk.gov.gchq.gaffer.spark.serialisation.kryo.Registrator").set("spark.driver.allowMultipleContexts", "true");
final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf);
// Create Hadoop configuration and serialise to a string
final Configuration configuration = new Configuration();
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
configuration.write(new DataOutputStream(baos));
final String configurationString = new String(baos.toByteArray(), CommonConstants.UTF_8);
// Check get correct edges for EdgeSeed 1 -> B
GetJavaRDDOfElements<EdgeSeed> rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext).seeds(Collections.singleton(new EdgeSeed("1", "B", false))).setIncludeEdges(GetOperation.IncludeEdgeType.ALL).setIncludeEntities(false).build();
rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString);
JavaRDD<Element> rdd = graph1.execute(rddQuery, user);
if (rdd == null) {
fail("No RDD returned");
}
final Set<Element> results = new HashSet<>();
results.addAll(rdd.collect());
final Set<Element> expectedElements = new HashSet<>();
final Edge edge1B = new Edge(EDGE_GROUP);
edge1B.setSource("1");
edge1B.setDestination("B");
edge1B.setDirected(false);
edge1B.putProperty("count", 2);
expectedElements.add(edge1B);
assertEquals(expectedElements, results);
// Check get entity for 1 when query for 1 -> B and specify entities only
rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext).seeds(Collections.singleton(new EdgeSeed("1", "B", false))).setIncludeEntities(true).setIncludeEdges(GetOperation.IncludeEdgeType.NONE).build();
rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString);
rdd = graph1.execute(rddQuery, user);
if (rdd == null) {
fail("No RDD returned");
}
results.clear();
results.addAll(rdd.collect());
expectedElements.clear();
final Entity entity1 = new Entity(ENTITY_GROUP);
entity1.setVertex("1");
expectedElements.add(entity1);
assertEquals(expectedElements, results);
// Check get correct edges for 1 -> B when specify edges only
rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext).seeds(Collections.singleton(new EdgeSeed("1", "B", false))).view(new View.Builder().edge(EDGE_GROUP).build()).setIncludeEntities(false).setIncludeEdges(GetOperation.IncludeEdgeType.ALL).build();
rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString);
rdd = graph1.execute(rddQuery, user);
if (rdd == null) {
fail("No RDD returned");
}
results.clear();
results.addAll(rdd.collect());
expectedElements.clear();
expectedElements.add(edge1B);
assertEquals(expectedElements, results);
// Check get correct edges for 1 -> B and 5 -> C
Set<EdgeSeed> seeds = new HashSet<>();
seeds.add(new EdgeSeed("1", "B", false));
seeds.add(new EdgeSeed("5", "C", false));
rddQuery = new GetJavaRDDOfElements.Builder<EdgeSeed>().javaSparkContext(sparkContext).setIncludeEntities(false).seeds(seeds).build();
rddQuery.addOption(AbstractGetRDDHandler.HADOOP_CONFIGURATION_KEY, configurationString);
rdd = graph1.execute(rddQuery, user);
if (rdd == null) {
fail("No RDD returned");
}
results.clear();
results.addAll(rdd.collect());
final Edge edge5C = new Edge(EDGE_GROUP);
edge5C.setSource("5");
edge5C.setDestination("C");
edge5C.setDirected(false);
edge5C.putProperty("count", 4);
expectedElements.clear();
expectedElements.add(edge1B);
expectedElements.add(edge5C);
assertEquals(expectedElements, results);
sparkContext.stop();
}
Aggregations