Examples with IteratorSetting org.apache.accumulo.core.client.IteratorSetting used on opensource projects

Example 1 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testGreaterThan1Sid.

@Test
public void testGreaterThan1Sid() throws Exception {
    Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
    IteratorSetting is = new IteratorSetting(1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, GreaterThan.class.getName());
    is.addOption(PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseIntBytes("1"))));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:sid");
    scan.addScanIterator(is);
    boolean foundMark = false;
    boolean foundDennis = false;
    int totalCount = 0;
    for (Map.Entry<Key, Value> kv : scan) {
        boolean foundName = false;
        boolean foundSid = false;
        boolean foundDegrees = false;
        boolean foundMillis = false;
        SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
        for (Map.Entry<Key, Value> item : items.entrySet()) {
            if (item.getKey().getRow().toString().equals("r2")) {
                foundMark = true;
            } else if (item.getKey().getRow().toString().equals("r3")) {
                foundDennis = true;
            }
            if (item.getKey().getColumnQualifier().equals(NAME)) {
                foundName = true;
            } else if (item.getKey().getColumnQualifier().equals(SID)) {
                foundSid = true;
            } else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
                foundDegrees = true;
            } else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
                foundMillis = true;
            }
        }
        totalCount++;
        assertTrue(foundDegrees & foundMillis & foundName & foundSid);
    }
    assertTrue(foundDennis & foundMark);
    assertEquals(totalCount, 2);
}

Example 2 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testNameEqualBrian.

@Test
public void testNameEqualBrian() throws Exception {
    Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
    IteratorSetting is = new IteratorSetting(1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    is.addOption(PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64("brian".getBytes())));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:name");
    scan.addScanIterator(is);
    boolean foundName = false;
    boolean foundSid = false;
    boolean foundDegrees = false;
    boolean foundMillis = false;
    for (Map.Entry<Key, Value> kv : scan) {
        SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
        for (Map.Entry<Key, Value> item : items.entrySet()) {
            assertEquals(item.getKey().getRow().toString(), "r1");
            if (item.getKey().getColumnQualifier().equals(NAME)) {
                foundName = true;
                assertArrayEquals(item.getValue().get(), "brian".getBytes());
            } else if (item.getKey().getColumnQualifier().equals(SID)) {
                foundSid = true;
                assertArrayEquals(item.getValue().get(), parseIntBytes("1"));
            } else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
                foundDegrees = true;
                assertArrayEquals(item.getValue().get(), parseDoubleBytes("44.5"));
            } else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
                foundMillis = true;
                assertArrayEquals(item.getValue().get(), parseLongBytes("555"));
            }
        }
    }
    assertTrue(foundDegrees & foundMillis & foundName & foundSid);
}

Example 3 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testConfigureAccumuloInputFormat.

@Test
public void testConfigureAccumuloInputFormat() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS), conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    Set<Pair<Text, Text>> cfCqPairs = inputformat.getPairCollection(columnMapper.getColumnMappings());
    List<IteratorSetting> iterators = Collections.emptyList();
    Set<Range> ranges = Collections.singleton(new Range());
    String instanceName = "realInstance";
    String zookeepers = "host1:2181,host2:2181,host3:2181";
    ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
    HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
    // Stub out the ZKI mock
    Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
    Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
    // Call out to the real configure method
    Mockito.doCallRealMethod().when(mockInputFormat).configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Also compute the correct cf:cq pairs so we can assert the right argument was passed
    Mockito.doCallRealMethod().when(mockInputFormat).getPairCollection(columnMapper.getColumnMappings());
    mockInputFormat.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Verify that the correct methods are invoked on AccumuloInputFormat
    Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
    Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
    Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
    Mockito.verify(mockInputFormat).setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
    Mockito.verify(mockInputFormat).addIterators(conf, iterators);
    Mockito.verify(mockInputFormat).setRanges(conf, ranges);
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}

Example 4 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testConfigureAccumuloInputFormatWithIterators.

@Test
public void testConfigureAccumuloInputFormatWithIterators() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS), conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    Set<Pair<Text, Text>> cfCqPairs = inputformat.getPairCollection(columnMapper.getColumnMappings());
    List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
    Set<Range> ranges = Collections.singleton(new Range());
    String instanceName = "realInstance";
    String zookeepers = "host1:2181,host2:2181,host3:2181";
    IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
    cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave");
    cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name");
    iterators.add(cfg);
    cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
    cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50");
    cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age");
    iterators.add(cfg);
    ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
    HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
    // Stub out the ZKI mock
    Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
    Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
    // Call out to the real configure method
    Mockito.doCallRealMethod().when(mockInputFormat).configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Also compute the correct cf:cq pairs so we can assert the right argument was passed
    Mockito.doCallRealMethod().when(mockInputFormat).getPairCollection(columnMapper.getColumnMappings());
    mockInputFormat.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
    // Verify that the correct methods are invoked on AccumuloInputFormat
    Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
    Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
    Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
    Mockito.verify(mockInputFormat).setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
    Mockito.verify(mockInputFormat).addIterators(conf, iterators);
    Mockito.verify(mockInputFormat).setRanges(conf, ranges);
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}

Example 5 with IteratorSetting

use of org.apache.accumulo.core.client.IteratorSetting in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testIteratorNotInSplitsCompensation.

@Test
public void testIteratorNotInSplitsCompensation() throws Exception {
    FileInputFormat.addInputPath(conf, new Path("unused"));
    InputSplit[] splits = inputformat.getSplits(conf, 0);
    assertEquals(1, splits.length);
    InputSplit split = splits[0];
    IteratorSetting is = new IteratorSetting(1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    is.addOption(PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(new byte[] { '0' })));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:cq");
    // Mock out the predicate handler because it's just easier
    AccumuloPredicateHandler predicateHandler = Mockito.mock(AccumuloPredicateHandler.class);
    Mockito.when(predicateHandler.getIterators(Mockito.any(JobConf.class), Mockito.any(ColumnMapper.class))).thenReturn(Arrays.asList(is));
    // Set it on our inputformat
    inputformat.predicateHandler = predicateHandler;
    inputformat.getRecordReader(split, conf, null);
    // The code should account for the bug and update the iterators on the split
    List<IteratorSetting> settingsOnSplit = ((HiveAccumuloSplit) split).getSplit().getIterators();
    assertEquals(1, settingsOnSplit.size());
    assertEquals(is, settingsOnSplit.get(0));
}