Examples with Prospector org.apache.rya.prospector.mr.Prospector used on opensource projects

Example 1 with Prospector

use of org.apache.rya.prospector.mr.Prospector in project incubator-rya by apache.

the class ProspectorServiceEvalStatsDAOTest method testNoAuthsCount.

@Test
public void testNoAuthsCount() throws Exception {
    // Load some data into a mock Accumulo and run the Prospector MapReduce job.
    final Instance mock = new MockInstance("accumulo");
    final Connector connector = mock.getConnector("user", new PasswordToken("pass"));
    final String outtable = "rya_prospects";
    if (connector.tableOperations().exists(outtable)) {
        connector.tableOperations().delete(outtable);
    }
    connector.tableOperations().create(outtable);
    connector.securityOperations().createUser("user", "pass".getBytes(), new Authorizations("U", "FOUO"));
    final AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO();
    ryaDAO.setConnector(connector);
    ryaDAO.init();
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12")));
    final String confFile = "stats_cluster_config.xml";
    final Path confPath = new Path(getClass().getClassLoader().getResource(confFile).toString());
    final String[] args = { confPath.toString() };
    ToolRunner.run(new Prospector(), args);
    ryaDAO.destroy();
    final Configuration conf = new Configuration();
    conf.addResource(confPath);
    final AccumuloRdfConfiguration rdfConf = new AccumuloRdfConfiguration(conf);
    final ProspectorServiceEvalStatsDAO evalDao = new ProspectorServiceEvalStatsDAO(connector, rdfConf);
    evalDao.init();
    // Get the cardinality of the 'urn:gem#pred' predicate.
    List<Value> values = new ArrayList<Value>();
    values.add(new URIImpl("urn:gem#pred"));
    double count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.PREDICATE, values);
    assertEquals(4.0, count, 0.001);
    // Get the cardinality of the 'mydata1' object.
    values = new ArrayList<Value>();
    values.add(new LiteralImpl("mydata1"));
    count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
    assertEquals(1.0, count, 0.001);
    // Get the cardinality of the 'mydata3' object.
    values = new ArrayList<Value>();
    values.add(new LiteralImpl("mydata3"));
    count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
    assertEquals(-1.0, count, 0.001);
}

Example 2 with Prospector

use of org.apache.rya.prospector.mr.Prospector in project incubator-rya by apache.

the class ProspectorServiceEvalStatsDAOTest method testCount.

@Test
public void testCount() throws Exception {
    // Load some data into a mock Accumulo and run the Prospector MapReduce job.
    final Instance mock = new MockInstance("accumulo");
    final Connector connector = mock.getConnector("user", new PasswordToken("pass"));
    final String outtable = "rya_prospects";
    if (connector.tableOperations().exists(outtable)) {
        connector.tableOperations().delete(outtable);
    }
    connector.tableOperations().create(outtable);
    final AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO();
    ryaDAO.setConnector(connector);
    ryaDAO.init();
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12")));
    final String confFile = "stats_cluster_config.xml";
    final Path confPath = new Path(getClass().getClassLoader().getResource(confFile).toString());
    final String[] args = { confPath.toString() };
    ToolRunner.run(new Prospector(), args);
    ryaDAO.destroy();
    final Configuration conf = new Configuration();
    conf.addResource(confPath);
    final AccumuloRdfConfiguration rdfConf = new AccumuloRdfConfiguration(conf);
    rdfConf.setAuths("U", "FOUO");
    final ProspectorServiceEvalStatsDAO evalDao = new ProspectorServiceEvalStatsDAO(connector, rdfConf);
    evalDao.init();
    // Get the cardinality of the 'urn:gem#pred' predicate.
    List<Value> values = new ArrayList<Value>();
    values.add(new URIImpl("urn:gem#pred"));
    double count = evalDao.getCardinality(rdfConf, CARDINALITY_OF.PREDICATE, values);
    assertEquals(4.0, count, 0.001);
    // Get the cardinality of the 'mydata1' object.
    values = new ArrayList<Value>();
    values.add(new LiteralImpl("mydata1"));
    count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
    assertEquals(1.0, count, 0.001);
    // Get the cardinality of the 'mydata3' object.
    values = new ArrayList<Value>();
    values.add(new LiteralImpl("mydata3"));
    count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
    assertEquals(-1.0, count, 0.001);
}

Example 3 with Prospector

use of org.apache.rya.prospector.mr.Prospector in project incubator-rya by apache.

the class ProspectorExample method main.

public static void main(final String[] args) throws Exception {
    setupLogging();
    // Configure Rya to use a mock instance.
    final AccumuloRdfConfiguration config = new AccumuloRdfConfiguration();
    config.useMockInstance(true);
    config.setTablePrefix("rya_");
    config.setUsername("user");
    config.setPassword("pass");
    config.setInstanceName("accumulo");
    // Load some data into Rya.
    final List<Statement> statements = Lists.newArrayList(VALUE_FACTORY.createStatement(ALICE, WORKS_AT, BURGER_JOINT), VALUE_FACTORY.createStatement(ALICE, ADMIRES, BOB), VALUE_FACTORY.createStatement(BOB, WORKS_AT, DONUT_SHOP), VALUE_FACTORY.createStatement(CHARLIE, WORKS_AT, DONUT_SHOP), VALUE_FACTORY.createStatement(CHARLIE, LIVES_WITH, BOB), VALUE_FACTORY.createStatement(BOB, LIVES_WITH, CHARLIE), VALUE_FACTORY.createStatement(BOB, LIVES_WITH, ALICE));
    final Sail sail = RyaSailFactory.getInstance(config);
    final SailConnection conn = sail.getConnection();
    log.info("Loading the following statements into a Mock instance of Accumulo Rya:");
    conn.begin();
    for (final Statement statement : statements) {
        log.info("    " + statement.toString());
        conn.addStatement(statement.getSubject(), statement.getPredicate(), statement.getObject());
    }
    conn.commit();
    conn.close();
    // Create the table that the Prospector's results will be written to.
    ConnectorFactory.connect(config).tableOperations().create("rya_prospects");
    // Run the Prospector using the configuration file that is in the resources directory.
    log.info("");
    log.info("Running the Map Reduce job that computes the Prospector results.");
    ToolRunner.run(new Prospector(), new String[] { "src/main/resources/stats_cluster_config.xml" });
    // Print the table that was created by the Prospector.
    log.info("");
    log.info("The following cardinalities were written to the Prospector table:");
    final ProspectorServiceEvalStatsDAO dao = ProspectorServiceEvalStatsDAO.make(config);
    // Do each of the Subjects.
    double cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECT, Lists.newArrayList(ALICE));
    log.info("    subject: " + ALICE + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECT, Lists.newArrayList(BOB));
    log.info("    subject: " + BOB + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECT, Lists.newArrayList(CHARLIE));
    log.info("    subject: " + CHARLIE + ", cardinality: " + cardinality);
    // Do each of the Predicates.
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATE, Lists.newArrayList(WORKS_AT));
    log.info("    predicate: " + WORKS_AT + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATE, Lists.newArrayList(ADMIRES));
    log.info("    predicate: " + ADMIRES + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATE, Lists.newArrayList(LIVES_WITH));
    log.info("    predicate: " + LIVES_WITH + ", cardinality: " + cardinality);
    // Do each of the Objects.
    cardinality = dao.getCardinality(config, CARDINALITY_OF.OBJECT, Lists.newArrayList(BURGER_JOINT));
    log.info("    object: " + BURGER_JOINT + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.OBJECT, Lists.newArrayList(DONUT_SHOP));
    log.info("    object: " + DONUT_SHOP + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.OBJECT, Lists.newArrayList(ALICE));
    log.info("    object: " + ALICE + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.OBJECT, Lists.newArrayList(BOB));
    log.info("    object: " + BOB + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.OBJECT, Lists.newArrayList(CHARLIE));
    log.info("    object: " + CHARLIE + ", cardinality: " + cardinality);
    // Do each of the Subject/Predicate pairs.
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTPREDICATE, Lists.newArrayList(ALICE, WORKS_AT));
    log.info("    subject/predicate: " + ALICE + "/" + WORKS_AT + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTPREDICATE, Lists.newArrayList(ALICE, ADMIRES));
    log.info("    subject/predicate: " + ALICE + "/" + ADMIRES + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTPREDICATE, Lists.newArrayList(BOB, WORKS_AT));
    log.info("    subject/predicate: " + BOB + "/" + WORKS_AT + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTPREDICATE, Lists.newArrayList(CHARLIE, WORKS_AT));
    log.info("    subject/predicate: " + CHARLIE + "/" + WORKS_AT + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTPREDICATE, Lists.newArrayList(CHARLIE, LIVES_WITH));
    log.info("    subject/predicate: " + CHARLIE + "/" + LIVES_WITH + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTPREDICATE, Lists.newArrayList(BOB, LIVES_WITH));
    log.info("    subject/predicate: " + BOB + "/" + LIVES_WITH + ", cardinality: " + cardinality);
    // Do each of the Subject/Object pairs.
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTOBJECT, Lists.newArrayList(ALICE, BURGER_JOINT));
    log.info("    subject/object: " + ALICE + "/" + BURGER_JOINT + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTOBJECT, Lists.newArrayList(ALICE, BOB));
    log.info("    subject/object: " + ALICE + "/" + BOB + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTOBJECT, Lists.newArrayList(BOB, DONUT_SHOP));
    log.info("    subject/object: " + ALICE + "/" + DONUT_SHOP + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTOBJECT, Lists.newArrayList(CHARLIE, DONUT_SHOP));
    log.info("    subject/object: " + CHARLIE + "/" + DONUT_SHOP + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTOBJECT, Lists.newArrayList(CHARLIE, BOB));
    log.info("    subject/object: " + CHARLIE + "/" + BOB + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTOBJECT, Lists.newArrayList(BOB, CHARLIE));
    log.info("    subject/object: " + BOB + "/" + CHARLIE + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.SUBJECTOBJECT, Lists.newArrayList(BOB, ALICE));
    log.info("    subject/object: " + BOB + "/" + ALICE + ", cardinality: " + cardinality);
    // Do each of the Predicate/Object pairs.
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATEOBJECT, Lists.newArrayList(WORKS_AT, BURGER_JOINT));
    log.info("    predicate/object: " + WORKS_AT + "/" + BURGER_JOINT + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATEOBJECT, Lists.newArrayList(ADMIRES, BOB));
    log.info("    predicate/object: " + ADMIRES + "/" + BOB + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATEOBJECT, Lists.newArrayList(WORKS_AT, DONUT_SHOP));
    log.info("    predicate/object: " + WORKS_AT + "/" + DONUT_SHOP + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATEOBJECT, Lists.newArrayList(LIVES_WITH, BOB));
    log.info("    predicate/object: " + LIVES_WITH + "/" + BOB + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATEOBJECT, Lists.newArrayList(LIVES_WITH, CHARLIE));
    log.info("    predicate/object: " + LIVES_WITH + "/" + CHARLIE + ", cardinality: " + cardinality);
    cardinality = dao.getCardinality(config, CARDINALITY_OF.PREDICATEOBJECT, Lists.newArrayList(LIVES_WITH, ALICE));
    log.info("    predicate/object: " + LIVES_WITH + "/" + ALICE + ", cardinality: " + cardinality);
}