Search in sources :

Example 16 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class IteratorSettingBuilderTest method shouldAddCompactSchemaToIteratorSetting.

@Test
public void shouldAddCompactSchemaToIteratorSetting() throws Exception {
    // Given
    final IteratorSetting setting = mock(IteratorSetting.class);
    final Schema schema = mock(Schema.class);
    final String compactSchemaJson = "CompactSchema";
    given(schema.toCompactJson()).willReturn(compactSchemaJson.getBytes());
    // When
    new IteratorSettingBuilder(setting).schema(schema);
    // Then
    verify(setting).addOption(AccumuloStoreConstants.SCHEMA, compactSchemaJson);
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) Schema(uk.gov.gchq.gaffer.store.schema.Schema) Test(org.junit.Test)

Example 17 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class IteratorSettingBuilderTest method shouldAddCompactViewToIteratorSetting.

@Test
public void shouldAddCompactViewToIteratorSetting() throws Exception {
    // Given
    final IteratorSetting setting = mock(IteratorSetting.class);
    final View view = mock(View.class);
    final String compactSchemaJson = "CompactSchema";
    given(view.toCompactJson()).willReturn(compactSchemaJson.getBytes());
    // When
    new IteratorSettingBuilder(setting).view(view);
    // Then
    verify(setting).addOption(AccumuloStoreConstants.VIEW, compactSchemaJson);
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) View(uk.gov.gchq.gaffer.data.elementdefinition.view.View) Test(org.junit.Test)

Example 18 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project Gaffer by gchq.

the class AbstractGetRDDHandler method addIterators.

public void addIterators(final AccumuloStore accumuloStore, final Configuration conf, final User user, final GetElementsOperation<?, ?> operation) throws OperationException {
    try {
        // Update configuration with instance name, table name, zookeepers, and with view
        accumuloStore.updateConfiguration(conf, operation.getView(), user);
        // Add iterators based on operation-specific (i.e. not view related) options
        final IteratorSetting edgeEntityDirectionFilter = accumuloStore.getKeyPackage().getIteratorFactory().getEdgeEntityDirectionFilterIteratorSetting(operation);
        if (edgeEntityDirectionFilter != null) {
            InputConfigurator.addIterator(AccumuloInputFormat.class, conf, edgeEntityDirectionFilter);
        }
        final IteratorSetting queryTimeAggregator = accumuloStore.getKeyPackage().getIteratorFactory().getQueryTimeAggregatorIteratorSetting(operation.getView(), accumuloStore);
        if (queryTimeAggregator != null) {
            InputConfigurator.addIterator(AccumuloInputFormat.class, conf, queryTimeAggregator);
        }
    } catch (final StoreException | IteratorSettingException e) {
        throw new OperationException("Failed to update configuration", e);
    }
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) IteratorSettingException(uk.gov.gchq.gaffer.accumulostore.key.exception.IteratorSettingException) OperationException(uk.gov.gchq.gaffer.operation.OperationException) StoreException(uk.gov.gchq.gaffer.store.StoreException)

Example 19 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testConfigureAccumuloInputFormatWithAuthorizations.

@Test
public void testConfigureAccumuloInputFormatWithAuthorizations() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    conf.set(AccumuloSerDeParameters.AUTHORIZATIONS_KEY, "foo,bar");
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS), conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    Set<Pair<Text, Text>> cfCqPairs = inputformat.getPairCollection(columnMapper.getColumnMappings());
    List<IteratorSetting> iterators = Collections.emptyList();
    Set<Range> ranges = Collections.singleton(new Range());
    String instanceName = "realInstance";
    String zookeepers = "host1:2181,host2:2181,host3:2181";
    ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
    HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
    // Stub out the ZKI mock
    Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
    Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
    // Call out to the real configure method
    Mockito.doCallRealMethod().when(mockInputFormat).configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Also compute the correct cf:cq pairs so we can assert the right argument was passed
    Mockito.doCallRealMethod().when(mockInputFormat).getPairCollection(columnMapper.getColumnMappings());
    mockInputFormat.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Verify that the correct methods are invoked on AccumuloInputFormat
    Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
    Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
    Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
    Mockito.verify(mockInputFormat).setScanAuthorizations(conf, new Authorizations("foo,bar"));
    Mockito.verify(mockInputFormat).addIterators(conf, iterators);
    Mockito.verify(mockInputFormat).setRanges(conf, ranges);
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
Also used : Authorizations(org.apache.accumulo.core.security.Authorizations) Range(org.apache.accumulo.core.data.Range) ZooKeeperInstance(org.apache.accumulo.core.client.ZooKeeperInstance) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) AccumuloConnectionParameters(org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters) ColumnMapper(org.apache.hadoop.hive.accumulo.columns.ColumnMapper) Pair(org.apache.accumulo.core.util.Pair) Test(org.junit.Test)

Example 20 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testConfigureAccumuloInputFormatWithEmptyColumns.

@Test
public void testConfigureAccumuloInputFormatWithEmptyColumns() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS), conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    HashSet<Pair<Text, Text>> cfCqPairs = Sets.newHashSet();
    List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
    Set<Range> ranges = Collections.singleton(new Range());
    String instanceName = "realInstance";
    String zookeepers = "host1:2181,host2:2181,host3:2181";
    IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
    cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave");
    cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name");
    iterators.add(cfg);
    cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
    cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50");
    cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age");
    iterators.add(cfg);
    ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
    HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
    // Stub out the ZKI mock
    Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
    Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
    Mockito.when(mockInputFormat.getPairCollection(columnMapper.getColumnMappings())).thenReturn(cfCqPairs);
    // Call out to the real configure method
    Mockito.doCallRealMethod().when(mockInputFormat).configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Also compute the correct cf:cq pairs so we can assert the right argument was passed
    Mockito.doCallRealMethod().when(mockInputFormat).getPairCollection(columnMapper.getColumnMappings());
    mockInputFormat.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Verify that the correct methods are invoked on AccumuloInputFormat
    Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
    Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
    Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
    Mockito.verify(mockInputFormat).setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
    Mockito.verify(mockInputFormat).addIterators(conf, iterators);
    Mockito.verify(mockInputFormat).setRanges(conf, ranges);
// fetchColumns is not called because we had no columns to fetch
}
Also used : StringCompare(org.apache.hadoop.hive.accumulo.predicate.compare.StringCompare) ArrayList(java.util.ArrayList) Range(org.apache.accumulo.core.data.Range) ZooKeeperInstance(org.apache.accumulo.core.client.ZooKeeperInstance) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) PrimitiveComparisonFilter(org.apache.hadoop.hive.accumulo.predicate.PrimitiveComparisonFilter) GreaterThanOrEqual(org.apache.hadoop.hive.accumulo.predicate.compare.GreaterThanOrEqual) Equal(org.apache.hadoop.hive.accumulo.predicate.compare.Equal) IntCompare(org.apache.hadoop.hive.accumulo.predicate.compare.IntCompare) AccumuloConnectionParameters(org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters) ColumnMapper(org.apache.hadoop.hive.accumulo.columns.ColumnMapper) Pair(org.apache.accumulo.core.util.Pair) Test(org.junit.Test)

Aggregations

IteratorSetting (org.apache.accumulo.core.client.IteratorSetting)35 Test (org.junit.Test)16 Scanner (org.apache.accumulo.core.client.Scanner)13 Authorizations (org.apache.accumulo.core.security.Authorizations)13 Key (org.apache.accumulo.core.data.Key)11 Value (org.apache.accumulo.core.data.Value)11 PasswordToken (org.apache.accumulo.core.client.security.tokens.PasswordToken)10 ColumnMapper (org.apache.hadoop.hive.accumulo.columns.ColumnMapper)10 Entry (java.util.Map.Entry)9 BatchWriterConfig (org.apache.accumulo.core.client.BatchWriterConfig)8 Range (org.apache.accumulo.core.data.Range)8 Map (java.util.Map)7 AccumuloException (org.apache.accumulo.core.client.AccumuloException)7 TableNotFoundException (org.apache.accumulo.core.client.TableNotFoundException)7 Mutation (org.apache.accumulo.core.data.Mutation)7 ColumnVisibility (org.apache.accumulo.core.security.ColumnVisibility)7 AccumuloConnectionParameters (org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters)7 BatchWriter (org.apache.accumulo.core.client.BatchWriter)6 IteratorSettingBuilder (uk.gov.gchq.gaffer.accumulostore.utils.IteratorSettingBuilder)6 Edge (uk.gov.gchq.gaffer.data.element.Edge)6