Examples with Authorizations org.apache.accumulo.core.security.Authorizations used on opensource projects

Example 11 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project presto by prestodb.

the class AccumuloClient method getScanAuthorizations.

/**
     * Gets the scan authorizations to use for scanning tables.
     * <p>
     * In order of priority: session username authorizations, then table property, then the default connector auths.
     *
     * @param session Current session
     * @param schema Schema name
     * @param table Table Name
     * @return Scan authorizations
     * @throws AccumuloException If a generic Accumulo error occurs
     * @throws AccumuloSecurityException If a security exception occurs
     */
private Authorizations getScanAuthorizations(ConnectorSession session, String schema, String table) throws AccumuloException, AccumuloSecurityException {
    String sessionScanUser = AccumuloSessionProperties.getScanUsername(session);
    if (sessionScanUser != null) {
        Authorizations scanAuths = connector.securityOperations().getUserAuthorizations(sessionScanUser);
        LOG.debug("Using session scan auths for user %s: %s", sessionScanUser, scanAuths);
        return scanAuths;
    }
    AccumuloTable accumuloTable = this.getTable(new SchemaTableName(schema, table));
    if (accumuloTable == null) {
        throw new TableNotFoundException(new SchemaTableName(schema, table));
    }
    Optional<String> strAuths = accumuloTable.getScanAuthorizations();
    if (strAuths.isPresent()) {
        Authorizations scanAuths = new Authorizations(Iterables.toArray(COMMA_SPLITTER.split(strAuths.get()), String.class));
        LOG.debug("scan_auths table property set, using: %s", scanAuths);
        return scanAuths;
    }
    LOG.debug("scan_auths table property not set, using connector auths: %s", this.auths);
    return this.auths;
}

Example 12 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project presto by prestodb.

the class TestIndexer method testMutationIndexWithVisibilities.

@Test
public void testMutationIndexWithVisibilities() throws Exception {
    Instance inst = new MockInstance();
    Connector conn = inst.getConnector("root", new PasswordToken(""));
    conn.tableOperations().create(table.getFullTableName());
    conn.tableOperations().create(table.getIndexTableName());
    conn.tableOperations().create(table.getMetricsTableName());
    for (IteratorSetting s : Indexer.getMetricIterators(table)) {
        conn.tableOperations().attachIterator(table.getMetricsTableName(), s);
    }
    Indexer indexer = new Indexer(conn, new Authorizations(), table, new BatchWriterConfig());
    indexer.index(m1);
    indexer.index(m1v);
    indexer.flush();
    Scanner scan = conn.createScanner(table.getIndexTableName(), new Authorizations("private", "moreprivate"));
    scan.setRange(new Range());
    Iterator<Entry<Key, Value>> iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "");
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "private", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "moreprivate", "");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "private", "");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "moreprivate", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "moreprivate", "");
    assertFalse(iter.hasNext());
    scan.close();
    scan = conn.createScanner(table.getMetricsTableName(), new Authorizations("private", "moreprivate"));
    scan.setRange(new Range());
    iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "1");
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "private", "1");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "2");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row1");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "moreprivate", "1");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "private", "1");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "moreprivate", "1");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "moreprivate", "1");
    assertFalse(iter.hasNext());
    scan.close();
    indexer.index(m2);
    indexer.index(m2v);
    indexer.close();
    scan = conn.createScanner(table.getIndexTableName(), new Authorizations("private", "moreprivate"));
    scan.setRange(new Range());
    iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "");
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", "private", "");
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", "");
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row2", "private", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", "moreprivate", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", "");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row2", "moreprivate", "");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "row1", "private", "");
    assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", "");
    assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "row2", "moreprivate", "");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "row1", "moreprivate", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row1", "moreprivate", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", "");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "row2", "moreprivate", "");
    assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", "");
    assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "row2", "moreprivate", "");
    assertFalse(iter.hasNext());
    scan.close();
    scan = conn.createScanner(table.getMetricsTableName(), new Authorizations("private", "moreprivate"));
    scan.setRange(new Range());
    iter = scan.iterator();
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "2");
    assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "___card___", "private", "2");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___card___", "4");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___first_row___", "row1");
    assertKeyValuePair(iter.next(), Indexer.METRICS_TABLE_ROW_ID.array(), "___rows___", "___last_row___", "row2");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "2");
    assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "___card___", "moreprivate", "2");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "1");
    assertKeyValuePair(iter.next(), M1_FNAME_VALUE, "cf_firstname", "___card___", "private", "1");
    assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "1");
    assertKeyValuePair(iter.next(), M2_FNAME_VALUE, "cf_firstname", "___card___", "moreprivate", "1");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("def"), "cf_arr", "___card___", "moreprivate", "1");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "2");
    assertKeyValuePair(iter.next(), bytes("ghi"), "cf_arr", "___card___", "moreprivate", "2");
    assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "1");
    assertKeyValuePair(iter.next(), bytes("mno"), "cf_arr", "___card___", "moreprivate", "1");
    assertFalse(iter.hasNext());
    scan.close();
}

Example 13 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method shouldPartiallyAggregateColumnQualifierOverCQ1GroupBy.

public void shouldPartiallyAggregateColumnQualifierOverCQ1GroupBy(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    final String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("1");
        edge2.setDestination("2");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 2);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge2.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("1");
        edge3.setDestination("2");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 3);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge3.putProperty(AccumuloPropertyNames.COUNT, 1);
        //THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
        final Edge edge4 = new Edge(TestGroups.EDGE);
        edge4.setSource("1");
        edge4.setDestination("2");
        edge4.setDirected(true);
        edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
        edge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge4.putProperty(AccumuloPropertyNames.COUNT, 2);
        final Edge edge5 = new Edge(TestGroups.EDGE);
        edge5.setSource("1");
        edge5.setDestination("2");
        edge5.setDirected(true);
        edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
        edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
        edge5.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge5.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge5.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge5.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge5.putProperty(AccumuloPropertyNames.COUNT, 10);
        final Edge edge6 = new Edge(TestGroups.EDGE);
        edge6.setSource("1");
        edge6.setDestination("2");
        edge6.setDirected(true);
        edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 4);
        edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 6);
        edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge6.putProperty(AccumuloPropertyNames.COUNT, 5);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        final Key key4 = elementConverter.getKeysFromEdge(edge4).getFirst();
        final Key key5 = elementConverter.getKeysFromEdge(edge5).getFirst();
        final Key key6 = elementConverter.getKeysFromEdge(edge6).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
        final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge4.getProperties());
        final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge5.getProperties());
        final Value value6 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge6.getProperties());
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key2.getRow());
        m2.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key.getRow());
        m4.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
        final Mutation m6 = new Mutation(key.getRow());
        m6.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value6);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.addMutation(m6);
        writer.close();
        Edge expectedEdge1 = new Edge(TestGroups.EDGE);
        expectedEdge1.setSource("1");
        expectedEdge1.setDestination("2");
        expectedEdge1.setDirected(true);
        expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 6);
        expectedEdge1.putProperty(AccumuloPropertyNames.COUNT, 3);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_4, 0);
        Edge expectedEdge2 = new Edge(TestGroups.EDGE);
        expectedEdge2.setSource("1");
        expectedEdge2.setDestination("2");
        expectedEdge2.setDirected(true);
        expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
        expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
        expectedEdge2.putProperty(AccumuloPropertyNames.COUNT, 2);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
        Edge expectedEdge3 = new Edge(TestGroups.EDGE);
        expectedEdge3.setSource("1");
        expectedEdge3.setDestination("2");
        expectedEdge3.setDirected(true);
        expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
        expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
        expectedEdge3.putProperty(AccumuloPropertyNames.COUNT, 10);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        Edge expectedEdge4 = new Edge(TestGroups.EDGE);
        expectedEdge4.setSource("1");
        expectedEdge4.setDestination("2");
        expectedEdge4.setDirected(true);
        expectedEdge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 4);
        expectedEdge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 6);
        expectedEdge4.putProperty(AccumuloPropertyNames.COUNT, 5);
        expectedEdge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy(AccumuloPropertyNames.COLUMN_QUALIFIER).build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        Entry<Key, Value> entry = it.next();
        Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge1, readEdge);
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(6, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge2, readEdge);
        assertEquals(2, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(4, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
        assertEquals(2, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge3, readEdge);
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
        assertEquals(10, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge4, readEdge);
        assertEquals(4, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(6, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
        assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 14 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method shouldAggregatePropertiesOnlyWhenGroupByIsSetToCQ1CQ2.

public void shouldAggregatePropertiesOnlyWhenGroupByIsSetToCQ1CQ2(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    final String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("1");
        edge2.setDestination("2");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge2.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("1");
        edge3.setDestination("2");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge3.putProperty(AccumuloPropertyNames.COUNT, 1);
        //THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
        final Edge edge4 = new Edge(TestGroups.EDGE);
        edge4.setSource("1");
        edge4.setDestination("2");
        edge4.setDirected(true);
        edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge4.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
        edge4.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge4.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge4.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge4.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge4.putProperty(AccumuloPropertyNames.COUNT, 2);
        final Edge edge5 = new Edge(TestGroups.EDGE);
        edge5.setSource("1");
        edge5.setDestination("2");
        edge5.setDirected(true);
        edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
        edge5.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
        edge5.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge5.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge5.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge5.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge5.putProperty(AccumuloPropertyNames.COUNT, 10);
        final Edge edge6 = new Edge(TestGroups.EDGE);
        edge6.setSource("1");
        edge6.setDestination("2");
        edge6.setDirected(true);
        edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
        edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
        edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge6.putProperty(AccumuloPropertyNames.COUNT, 5);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        final Key key4 = elementConverter.getKeysFromEdge(edge4).getFirst();
        final Key key5 = elementConverter.getKeysFromEdge(edge5).getFirst();
        final Key key6 = elementConverter.getKeysFromEdge(edge6).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
        final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge4.getProperties());
        final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge5.getProperties());
        final Value value6 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge6.getProperties());
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key2.getRow());
        m2.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key.getRow());
        m4.put(key4.getColumnFamily(), key4.getColumnQualifier(), new ColumnVisibility(key4.getColumnVisibility()), key4.getTimestamp(), value4);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key5.getColumnFamily(), key5.getColumnQualifier(), new ColumnVisibility(key5.getColumnVisibility()), key5.getTimestamp(), value5);
        final Mutation m6 = new Mutation(key.getRow());
        m6.put(key6.getColumnFamily(), key6.getColumnQualifier(), new ColumnVisibility(key6.getColumnVisibility()), key6.getTimestamp(), value6);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.addMutation(m6);
        writer.close();
        Edge expectedEdge1 = new Edge(TestGroups.EDGE);
        expectedEdge1.setSource("1");
        expectedEdge1.setDestination("2");
        expectedEdge1.setDirected(true);
        expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        expectedEdge1.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 1);
        expectedEdge1.putProperty(AccumuloPropertyNames.COUNT, 3);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge1.putProperty(AccumuloPropertyNames.PROP_4, 0);
        Edge expectedEdge2 = new Edge(TestGroups.EDGE);
        expectedEdge2.setSource("1");
        expectedEdge2.setDestination("2");
        expectedEdge2.setDirected(true);
        expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        expectedEdge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 4);
        expectedEdge2.putProperty(AccumuloPropertyNames.COUNT, 2);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
        Edge expectedEdge3 = new Edge(TestGroups.EDGE);
        expectedEdge3.setSource("1");
        expectedEdge3.setDestination("2");
        expectedEdge3.setDirected(true);
        expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 3);
        expectedEdge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2, 5);
        expectedEdge3.putProperty(AccumuloPropertyNames.COUNT, 15);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy(AccumuloPropertyNames.COLUMN_QUALIFIER, AccumuloPropertyNames.COLUMN_QUALIFIER_2).build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        Entry<Key, Value> entry = it.next();
        Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge1, readEdge);
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge2, readEdge);
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(4, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
        assertEquals(2, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        entry = it.next();
        readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge3, readEdge);
        assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER_2));
        assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 15 with Authorizations

use of org.apache.accumulo.core.security.Authorizations in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingMultiplePropertySets.

private void testAggregatingMultiplePropertySets(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        //THIS EDGE WILL BE REDUCED MEANING ITS CQ (columnQualifier) will only occur once because its key is equal.
        final Edge edge2 = new Edge(TestGroups.EDGE);
        edge2.setSource("1");
        edge2.setDestination("2");
        edge2.setDirected(true);
        edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge2.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge2.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge2.putProperty(AccumuloPropertyNames.COUNT, 2);
        final Edge edge3 = new Edge(TestGroups.EDGE);
        edge3.setSource("1");
        edge3.setDestination("2");
        edge3.setDirected(true);
        edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
        edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
        edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
        edge3.putProperty(AccumuloPropertyNames.COUNT, 10);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
        final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge.getProperties());
        final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge2.getProperties());
        final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, edge3.getProperties());
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        final Mutation m2 = new Mutation(key.getRow());
        m2.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value2);
        final Mutation m3 = new Mutation(key.getRow());
        m3.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value3);
        final Mutation m4 = new Mutation(key2.getRow());
        m4.put(key2.getColumnFamily(), key2.getColumnQualifier(), new ColumnVisibility(key2.getColumnVisibility()), key2.getTimestamp(), value1);
        final Mutation m5 = new Mutation(key.getRow());
        m5.put(key3.getColumnFamily(), key3.getColumnQualifier(), new ColumnVisibility(key3.getColumnVisibility()), key3.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.addMutation(m2);
        writer.addMutation(m3);
        writer.addMutation(m4);
        writer.addMutation(m5);
        writer.close();
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().schema(store.getSchema()).view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        final Edge expectedEdge = new Edge(TestGroups.EDGE);
        expectedEdge.setSource("1");
        expectedEdge.setDestination("2");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 9);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 15);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_1, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_2, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_3, 0);
        expectedEdge.putProperty(AccumuloPropertyNames.PROP_4, 0);
        assertEquals(expectedEdge, readEdge);
        assertEquals(9, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(15, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}