Examples with Authorizations org.apache.accumulo.core.security.Authorizations used on opensource projects

Example 71 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project incubator-rya by apache.

the class JoinSelectStatisticsTest method testMap3.

@Test
public void testMap3() throws Exception {
    init();
    System.out.println("*************************Test3**************************** ");
    BatchWriter bw_table1 = c.createBatchWriter("rya_spo", new BatchWriterConfig());
    for (int i = 1; i < 3; i++) {
        for (int j = 1; j < 3; j++) {
            for (int k = 1; k < 3; k++) {
                RyaStatement rs = new RyaStatement(new RyaURI(uri + i), new RyaURI(uri + (j)), new RyaType(uri + k));
                Map<TABLE_LAYOUT, TripleRow> tripleRowMap = ryaContext.serializeTriple(rs);
                TripleRow tripleRow = tripleRowMap.get(TABLE_LAYOUT.SPO);
                Mutation m = JoinSelectStatsUtil.createMutation(tripleRow);
                bw_table1.addMutation(m);
            }
        }
    }
    bw_table1.close();
    BatchWriter bw_table2 = c.createBatchWriter("rya_prospects", new BatchWriterConfig());
    for (int i = 1; i < 3; i++) {
        int k = 1;
        for (String s : cardList) {
            Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + i));
            m.put(new Text(), new Text(), new Value(new IntWritable(i + k).toString().getBytes()));
            bw_table2.addMutation(m);
            k++;
        }
        for (int j = 1; j < 3; j++) {
            k = 1;
            for (String s : aggCardList) {
                Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + uri + j + DELIM + i));
                m.put(new Text(), new Text(), new Value(new IntWritable(i + k + j).toString().getBytes()));
                bw_table2.addMutation(m);
                k++;
            }
        }
    }
    bw_table2.close();
    Assert.assertEquals(0, ToolRunner.run(new JoinSelectTestDriver(), new String[] { "" }));
    Scanner scan = c.createScanner("rya_selectivity", new Authorizations());
    scan.setRange(new Range());
    for (Map.Entry<Key, Value> entry : scan) {
        System.out.println("Key row string is " + entry.getKey().getRow().toString());
        System.out.println("Join type is " + entry.getKey().getColumnFamily().toString());
        System.out.println("Value is " + entry.getKey().getColumnQualifier().toString());
    }
    Scanner scan1 = c.createScanner("rya_selectivity", new Authorizations());
    scan1.setRange(Range.prefix("subject" + DELIM + uri + 1));
    int i = 0;
    for (Map.Entry<Key, Value> entry : scan1) {
        Key key = entry.getKey();
        String s = key.getColumnFamily().toString();
        int val = Integer.parseInt(key.getColumnQualifier().toString());
        if (s.equals("predicatepredicate")) {
            Assert.assertTrue(val == 14);
        }
        if (s.equals("objectobject")) {
            Assert.assertTrue(val == 18);
        }
        if (s.equals("predicateobjectpredicateobject")) {
            Assert.assertTrue(val == 28);
        }
        if (s.equals("predicateobjectsubjectpredicate")) {
            Assert.assertTrue(val == 20);
        }
        if (s.equals("predicateobjectobjectsubject")) {
            Assert.assertTrue(val == 16);
        }
        i++;
    }
    Assert.assertTrue(i == 12);
}

Example 72 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project incubator-rya by apache.

the class ProspectorServiceEvalStatsDAOTest method testNoAuthsCount.

@Test
public void testNoAuthsCount() throws Exception {
    // Load some data into a mock Accumulo and run the Prospector MapReduce job.
    final Instance mock = new MockInstance("accumulo");
    final Connector connector = mock.getConnector("user", new PasswordToken("pass"));
    final String outtable = "rya_prospects";
    if (connector.tableOperations().exists(outtable)) {
        connector.tableOperations().delete(outtable);
    }
    connector.tableOperations().create(outtable);
    connector.securityOperations().createUser("user", "pass".getBytes(), new Authorizations("U", "FOUO"));
    final AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO();
    ryaDAO.setConnector(connector);
    ryaDAO.init();
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12")));
    ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12")));
    final String confFile = "stats_cluster_config.xml";
    final Path confPath = new Path(getClass().getClassLoader().getResource(confFile).toString());
    final String[] args = { confPath.toString() };
    ToolRunner.run(new Prospector(), args);
    ryaDAO.destroy();
    final Configuration conf = new Configuration();
    conf.addResource(confPath);
    final AccumuloRdfConfiguration rdfConf = new AccumuloRdfConfiguration(conf);
    final ProspectorServiceEvalStatsDAO evalDao = new ProspectorServiceEvalStatsDAO(connector, rdfConf);
    evalDao.init();
    // Get the cardinality of the 'urn:gem#pred' predicate.
    List<Value> values = new ArrayList<Value>();
    values.add(new URIImpl("urn:gem#pred"));
    double count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.PREDICATE, values);
    assertEquals(4.0, count, 0.001);
    // Get the cardinality of the 'mydata1' object.
    values = new ArrayList<Value>();
    values.add(new LiteralImpl("mydata1"));
    count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
    assertEquals(1.0, count, 0.001);
    // Get the cardinality of the 'mydata3' object.
    values = new ArrayList<Value>();
    values.add(new LiteralImpl("mydata3"));
    count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
    assertEquals(-1.0, count, 0.001);
}

Example 73 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project incubator-rya by apache.

the class AccumuloSelectivityEvalDAO method getJoinSelect.

private double getJoinSelect(RdfCloudTripleStoreConfiguration conf, StatementPattern sp1, StatementPattern sp2) throws TableNotFoundException {
    if (FullTableCardinality == 0) {
        this.getTableSize(conf);
    }
    Authorizations authorizations = getAuths(conf);
    String row1 = CardinalityCalcUtil.getRow(sp1, true);
    String row2 = CardinalityCalcUtil.getRow(sp2, true);
    List<String> joinType = CardinalityCalcUtil.getJoins(sp1, sp2);
    if (joinType.size() == 0) {
        return 1;
    }
    if (joinType.size() == 2) {
        String cacheRow1;
        String cacheRow2;
        long card1 = 0;
        long card2 = 0;
        boolean contCard1 = false;
        boolean contCard2 = false;
        cacheRow1 = row1 + DELIM + joinType.get(0);
        cacheRow2 = row2 + DELIM + joinType.get(1);
        long count1 = getCardinality(conf, sp1);
        long count2 = getCardinality(conf, sp2);
        if (count1 == 0 || count2 == 0) {
            return 0;
        }
        if (joinMap.containsKey(cacheRow1)) {
            card1 = joinMap.get(cacheRow1);
            contCard1 = true;
        }
        if (joinMap.containsKey(cacheRow2)) {
            card2 = joinMap.get(cacheRow2);
            contCard2 = true;
        }
        if (!contCard1) {
            Scanner joinScanner = connector.createScanner(tableLayoutStrategy.getSelectivity(), authorizations);
            joinScanner.setRange(Range.prefix(row1));
            for (Map.Entry<Key, Value> entry : joinScanner) {
                if (entry.getKey().getColumnFamily().toString().equals(joinType.get(0))) {
                    card1 = CardinalityCalcUtil.getJCard(entry.getKey());
                    joinMap.put(cacheRow1, card1);
                    // System.out.println("Card1 is " + card1);
                    break;
                }
            }
        }
        if (!contCard2) {
            Scanner joinScanner = connector.createScanner(tableLayoutStrategy.getSelectivity(), authorizations);
            joinScanner.setRange(Range.prefix(row2));
            for (Map.Entry<Key, Value> entry : joinScanner) {
                if (entry.getKey().getColumnFamily().toString().equals(joinType.get(1))) {
                    card2 = CardinalityCalcUtil.getJCard(entry.getKey());
                    joinMap.put(cacheRow2, card2);
                    // System.out.println("Card2 is " + card2);
                    break;
                }
            }
        }
        if (!filtered && !denormalized) {
            double temp1 = Math.min(((double) card1) / ((double) count1 * FullTableCardinality), ((double) card2) / ((double) count2 * FullTableCardinality));
            double temp2 = Math.max((double) count1 / FullTableCardinality, (double) count2 / FullTableCardinality);
            // TODO maybe change back to original form as temp2 will rarely be less than temp1.
            return Math.min(temp1, temp2);
        } else if (denormalized) {
            return Math.min(card1, card2);
        } else {
            return Math.min(((double) card1 * count2) / ((double) count1 * FullTableCardinality * FullTableCardinality), ((double) card2 * count1) / ((double) count2 * FullTableCardinality * FullTableCardinality));
        }
    } else {
        String cacheRow1 = row1 + DELIM + joinType.get(0);
        String cacheRow2 = row1 + DELIM + joinType.get(1);
        String cacheRow3 = row2 + DELIM + joinType.get(2);
        String cacheRow4 = row2 + DELIM + joinType.get(3);
        long card1 = 0;
        long card2 = 0;
        long card3 = 0;
        long card4 = 0;
        boolean contCard1 = false;
        boolean contCard2 = false;
        long count1 = getCardinality(conf, sp1);
        long count2 = getCardinality(conf, sp2);
        if (count1 == 0 || count2 == 0) {
            return 0;
        }
        if (joinMap.containsKey(cacheRow1) && joinMap.containsKey(cacheRow2)) {
            card1 = joinMap.get(cacheRow1);
            card2 = joinMap.get(cacheRow2);
            contCard1 = true;
        }
        if (joinMap.containsKey(cacheRow3) && joinMap.containsKey(cacheRow4)) {
            card3 = joinMap.get(cacheRow3);
            card4 = joinMap.get(cacheRow4);
            contCard2 = true;
        }
        if (!contCard1) {
            Scanner joinScanner = connector.createScanner(tableLayoutStrategy.getSelectivity(), authorizations);
            joinScanner.setRange(Range.prefix(row1));
            boolean found1 = false;
            boolean found2 = false;
            for (Map.Entry<Key, Value> entry : joinScanner) {
                if (entry.getKey().getColumnFamily().toString().equals(joinType.get(0))) {
                    card1 = CardinalityCalcUtil.getJCard(entry.getKey());
                    joinMap.put(cacheRow1, card1);
                    found1 = true;
                    // System.out.println("Card1 is " + card1);
                    if (found1 && found2) {
                        card1 = Math.min(card1, card2);
                        break;
                    }
                } else if (entry.getKey().getColumnFamily().toString().equals(joinType.get(1))) {
                    card2 = CardinalityCalcUtil.getJCard(entry.getKey());
                    joinMap.put(cacheRow2, card2);
                    found2 = true;
                    // System.out.println("Card1 is " + card1);
                    if (found1 && found2) {
                        card1 = Math.min(card1, card2);
                        break;
                    }
                }
            }
        }
        if (!contCard2) {
            Scanner joinScanner = connector.createScanner(tableLayoutStrategy.getSelectivity(), authorizations);
            joinScanner.setRange(Range.prefix(row2));
            boolean found1 = false;
            boolean found2 = false;
            for (Map.Entry<Key, Value> entry : joinScanner) {
                if (entry.getKey().getColumnFamily().toString().equals(joinType.get(2))) {
                    card3 = CardinalityCalcUtil.getJCard(entry.getKey());
                    joinMap.put(cacheRow3, card3);
                    found1 = true;
                    // System.out.println("Card2 is " + card2);
                    if (found1 && found2) {
                        card3 = Math.min(card3, card4);
                        break;
                    }
                } else if (entry.getKey().getColumnFamily().toString().equals(joinType.get(3))) {
                    card4 = CardinalityCalcUtil.getJCard(entry.getKey());
                    joinMap.put(cacheRow4, card4);
                    found2 = true;
                    // System.out.println("Card1 is " + card1);
                    if (found1 && found2) {
                        card3 = Math.min(card3, card4);
                        break;
                    }
                }
            }
        }
        if (!filtered && !denormalized) {
            return Math.min(((double) card1) / ((double) count1 * FullTableCardinality), ((double) card3) / ((double) count2 * FullTableCardinality));
        } else if (denormalized) {
            return Math.min(card1, card3);
        } else {
            return Math.min(((double) card1 * count2) / ((double) count1 * FullTableCardinality * FullTableCardinality), ((double) card3 * count1) / ((double) count2 * FullTableCardinality * FullTableCardinality));
        }
    }
}

Example 74 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project incubator-rya by apache.

the class AccumuloSelectivityEvalDAOTest method testJoinCardinalityQuery1.

@Test
public void testJoinCardinalityQuery1() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MalformedQueryException {
    AccumuloSelectivityEvalDAO accc = new AccumuloSelectivityEvalDAO();
    accc.setConf(arc);
    accc.setConnector(conn);
    accc.setRdfEvalDAO(res);
    accc.init();
    BatchWriter bw1 = conn.createBatchWriter("rya_prospects", config);
    BatchWriter bw2 = conn.createBatchWriter("rya_selectivity", config);
    String s1 = "predicateobject" + DELIM + "http://www.w3.org/2000/01/rdf-schema#label" + DELIM + "uri:dog";
    String s2 = "predicateobject" + DELIM + "uri:barksAt" + DELIM + "uri:cat";
    String s3 = "predicateobject" + DELIM + "uri:peesOn" + DELIM + "uri:hydrant";
    List<Mutation> mList = new ArrayList<Mutation>();
    List<Mutation> mList2 = new ArrayList<Mutation>();
    List<String> sList = Arrays.asList("subjectobject", "subjectpredicate", "subjectsubject", "predicateobject", "predicatepredicate", "predicatesubject");
    Mutation m1, m2, m3, m4;
    m1 = new Mutation(s1 + DELIM + "1");
    m1.put(new Text("count"), new Text(""), new Value("20".getBytes()));
    m2 = new Mutation(s2 + DELIM + "2");
    m2.put(new Text("count"), new Text(""), new Value("15".getBytes()));
    m3 = new Mutation(s3 + DELIM + "3");
    m3.put(new Text("count"), new Text(""), new Value("10".getBytes()));
    mList.add(m1);
    mList.add(m2);
    mList.add(m3);
    bw1.addMutations(mList);
    bw1.close();
    m1 = new Mutation(s1);
    m2 = new Mutation(s2);
    m3 = new Mutation(s3);
    int i = 30;
    int j = 60;
    int k = 90;
    Long count1;
    Long count2;
    Long count3;
    for (String s : sList) {
        count1 = (long) i;
        count2 = (long) j;
        count3 = (long) k;
        m1.put(new Text(s), new Text(count1.toString()), EMPTY_VAL);
        m2.put(new Text(s), new Text(count2.toString()), EMPTY_VAL);
        m3.put(new Text(s), new Text(count3.toString()), EMPTY_VAL);
        i = 2 * i;
        j = 2 * j;
        k = 2 * k;
    }
    m4 = new Mutation(new Text("subjectpredicateobject" + DELIM + "FullTableCardinality"));
    m4.put(new Text("FullTableCardinality"), new Text("600"), EMPTY_VAL);
    mList2.add(m1);
    mList2.add(m2);
    mList2.add(m3);
    mList2.add(m4);
    bw2.addMutations(mList2);
    bw2.close();
    Scanner scan = conn.createScanner("rya_selectivity", new Authorizations());
    scan.setRange(new Range());
    for (Map.Entry<Key, Value> entry : scan) {
        System.out.println("Key row string is " + entry.getKey().getRow().toString());
        System.out.println("Key is " + entry.getKey());
        System.out.println("Value is " + (new String(entry.getKey().getColumnQualifier().toString())));
    }
    List<StatementPattern> spList = getSpList(q1);
    System.out.println(spList);
    List<Double> jCardList = new ArrayList<Double>();
    for (StatementPattern sp1 : spList) {
        for (StatementPattern sp2 : spList) {
            jCardList.add(accc.getJoinSelect(arc, sp1, sp2));
        }
    }
    System.out.println("Join cardinalities are " + jCardList);
    Assert.assertEquals(0, jCardList.get(0), .001);
    Assert.assertEquals(0, jCardList.get(3), .001);
    Assert.assertEquals(6.0 / 600, jCardList.get(5), .001);
    Assert.assertEquals(6.0 / 600, jCardList.get(6), .001);
    Assert.assertEquals(0 / 600, jCardList.get(8), .001);
    Assert.assertEquals(6.0 / 600, jCardList.get(7), .001);
    Assert.assertEquals(15.0 / 600, jCardList.get(11), .001);
    Assert.assertEquals(6.0 / 600, jCardList.get(13), .001);
    Assert.assertEquals(10.0 / 600, jCardList.get(15), .001);
    Assert.assertTrue(jCardList.get(0) == 0);
    Assert.assertTrue(jCardList.get(3) == 0);
    Assert.assertTrue(jCardList.get(5) == .01);
    Assert.assertTrue(jCardList.get(6) == .01);
    Assert.assertTrue(jCardList.get(8) == 0);
    Assert.assertTrue(jCardList.get(7) == (6.0 / 600));
    Assert.assertTrue(jCardList.get(11) == (1.0 / 40));
    Assert.assertTrue(jCardList.get(13) == .01);
    Assert.assertTrue(jCardList.get(15) == (10.0 / 600));
}

Example 75 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project incubator-rya by apache.

the class JoinSelectStatsUtil method initTableMRJob.

public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException {
    Configuration conf = job.getConfiguration();
    String username = conf.get(USERNAME);
    String password = conf.get(PASSWORD);
    String instance = conf.get(INSTANCE);
    String zookeepers = conf.get(ZOOKEEPERS);
    System.out.println("Zookeepers are " + auths);
    if (zookeepers != null) {
        AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers);
        AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers);
    } else {
        throw new IllegalArgumentException("Must specify either mock or zookeepers");
    }
    AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths));
    AccumuloInputFormat.setInputTableName(job, intable);
    job.setInputFormatClass(AccumuloInputFormat.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    // OUTPUT
    AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password));
    AccumuloOutputFormat.setDefaultTableName(job, outtable);
    job.setOutputFormatClass(AccumuloOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Mutation.class);
}