Examples with SortedMap java.util.SortedMap used on opensource projects

Example 11 with SortedMap

use of java.util.SortedMap in project hive by apache.

the class HiveRelDecorrelator method decorrelateRel.

/**
   * Rewrite LogicalJoin.
   *
   * @param rel LogicalJoin
   */
public Frame decorrelateRel(LogicalJoin rel) {
    //
    // Rewrite logic:
    //
    // 1. rewrite join condition.
    // 2. map output positions and produce cor vars if any.
    //
    final RelNode oldLeft = rel.getInput(0);
    final RelNode oldRight = rel.getInput(1);
    final Frame leftFrame = getInvoke(oldLeft, rel);
    final Frame rightFrame = getInvoke(oldRight, rel);
    if (leftFrame == null || rightFrame == null) {
        // If any input has not been rewritten, do not rewrite this rel.
        return null;
    }
    final RelNode newJoin = HiveJoin.getJoin(rel.getCluster(), leftFrame.r, rightFrame.r, decorrelateExpr(rel.getCondition()), rel.getJoinType());
    // Create the mapping between the output of the old correlation rel
    // and the new join rel
    Map<Integer, Integer> mapOldToNewOutputs = Maps.newHashMap();
    int oldLeftFieldCount = oldLeft.getRowType().getFieldCount();
    int newLeftFieldCount = leftFrame.r.getRowType().getFieldCount();
    int oldRightFieldCount = oldRight.getRowType().getFieldCount();
    assert rel.getRowType().getFieldCount() == oldLeftFieldCount + oldRightFieldCount;
    // Left input positions are not changed.
    mapOldToNewOutputs.putAll(leftFrame.oldToNewOutputs);
    // Right input positions are shifted by newLeftFieldCount.
    for (int i = 0; i < oldRightFieldCount; i++) {
        mapOldToNewOutputs.put(i + oldLeftFieldCount, rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount);
    }
    final SortedMap<CorDef, Integer> corDefOutputs = new TreeMap<>(leftFrame.corDefOutputs);
    // Right input positions are shifted by newLeftFieldCount.
    for (Map.Entry<CorDef, Integer> entry : rightFrame.corDefOutputs.entrySet()) {
        corDefOutputs.put(entry.getKey(), entry.getValue() + newLeftFieldCount);
    }
    return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs);
}

Example 12 with SortedMap

use of java.util.SortedMap in project hbase by apache.

the class TestCatalogJanitor method parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst.

/**
   * Make sure parent with specified end key gets cleaned up even if daughter is cleaned up before it.
   *
   * @param rootDir the test case name, used as the HBase testing utility root
   * @param lastEndKey the end key of the split parent
   * @throws IOException
   * @throws InterruptedException
   */
private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(final String rootDir, final byte[] lastEndKey) throws IOException, InterruptedException {
    HBaseTestingUtility htu = new HBaseTestingUtility();
    setRootDirAndCleanIt(htu, rootDir);
    MasterServices services = new MockMasterServices(htu);
    CatalogJanitor janitor = new CatalogJanitor(services);
    final HTableDescriptor htd = createHTableDescriptor();
    // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc.
    // Parent
    HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), lastEndKey);
    // Sleep a second else the encoded name on these regions comes out
    // same for all with same start key and made in same second.
    Thread.sleep(1001);
    // Daughter a
    HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
    Thread.sleep(1001);
    // Make daughters of daughter a; splitaa and splitab.
    HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"));
    HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"));
    // Daughter b
    HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), lastEndKey);
    Thread.sleep(1001);
    // Make Daughters of daughterb; splitba and splitbb.
    HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"));
    HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"), lastEndKey);
    // First test that our Comparator works right up in CatalogJanitor.
    // Just fo kicks.
    SortedMap<HRegionInfo, Result> regions = new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator());
    // Now make sure that this regions map sorts as we expect it to.
    regions.put(parent, createResult(parent, splita, splitb));
    regions.put(splitb, createResult(splitb, splitba, splitbb));
    regions.put(splita, createResult(splita, splitaa, splitab));
    // Assert its properly sorted.
    int index = 0;
    for (Map.Entry<HRegionInfo, Result> e : regions.entrySet()) {
        if (index == 0) {
            assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
        } else if (index == 1) {
            assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
        } else if (index == 2) {
            assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
        }
        index++;
    }
    // Now play around with the cleanParent function.  Create a ref from splita
    // up to the parent.
    Path splitaRef = createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
    // Make sure actual super parent sticks around because splita has a ref.
    assertFalse(janitor.cleanParent(parent, regions.get(parent)));
    //splitba, and split bb, do not have dirs in fs.  That means that if
    // we test splitb, it should get cleaned up.
    assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
    // Now remove ref from splita to parent... so parent can be let go and so
    // the daughter splita can be split (can't split if still references).
    // BUT make the timing such that the daughter gets cleaned up before we
    // can get a chance to let go of the parent.
    FileSystem fs = FileSystem.get(htu.getConfiguration());
    assertTrue(fs.delete(splitaRef, true));
    // Create the refs from daughters of splita.
    Path splitaaRef = createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
    Path splitabRef = createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
    // Test splita.  It should stick around because references from splitab, etc.
    assertFalse(janitor.cleanParent(splita, regions.get(splita)));
    // Now clean up parent daughter first.  Remove references from its daughters.
    assertTrue(fs.delete(splitaaRef, true));
    assertTrue(fs.delete(splitabRef, true));
    assertTrue(janitor.cleanParent(splita, regions.get(splita)));
    // Super parent should get cleaned up now both splita and splitb are gone.
    assertTrue(janitor.cleanParent(parent, regions.get(parent)));
    services.stop("test finished");
    janitor.cancel(true);
}

Example 13 with SortedMap

use of java.util.SortedMap in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testGreaterThan1Sid.

@Test
public void testGreaterThan1Sid() throws Exception {
    Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
    IteratorSetting is = new IteratorSetting(1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, GreaterThan.class.getName());
    is.addOption(PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseIntBytes("1"))));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:sid");
    scan.addScanIterator(is);
    boolean foundMark = false;
    boolean foundDennis = false;
    int totalCount = 0;
    for (Map.Entry<Key, Value> kv : scan) {
        boolean foundName = false;
        boolean foundSid = false;
        boolean foundDegrees = false;
        boolean foundMillis = false;
        SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
        for (Map.Entry<Key, Value> item : items.entrySet()) {
            if (item.getKey().getRow().toString().equals("r2")) {
                foundMark = true;
            } else if (item.getKey().getRow().toString().equals("r3")) {
                foundDennis = true;
            }
            if (item.getKey().getColumnQualifier().equals(NAME)) {
                foundName = true;
            } else if (item.getKey().getColumnQualifier().equals(SID)) {
                foundSid = true;
            } else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
                foundDegrees = true;
            } else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
                foundMillis = true;
            }
        }
        totalCount++;
        assertTrue(foundDegrees & foundMillis & foundName & foundSid);
    }
    assertTrue(foundDennis & foundMark);
    assertEquals(totalCount, 2);
}

Example 14 with SortedMap

use of java.util.SortedMap in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testNameEqualBrian.

@Test
public void testNameEqualBrian() throws Exception {
    Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
    IteratorSetting is = new IteratorSetting(1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    is.addOption(PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64("brian".getBytes())));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:name");
    scan.addScanIterator(is);
    boolean foundName = false;
    boolean foundSid = false;
    boolean foundDegrees = false;
    boolean foundMillis = false;
    for (Map.Entry<Key, Value> kv : scan) {
        SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
        for (Map.Entry<Key, Value> item : items.entrySet()) {
            assertEquals(item.getKey().getRow().toString(), "r1");
            if (item.getKey().getColumnQualifier().equals(NAME)) {
                foundName = true;
                assertArrayEquals(item.getValue().get(), "brian".getBytes());
            } else if (item.getKey().getColumnQualifier().equals(SID)) {
                foundSid = true;
                assertArrayEquals(item.getValue().get(), parseIntBytes("1"));
            } else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
                foundDegrees = true;
                assertArrayEquals(item.getValue().get(), parseDoubleBytes("44.5"));
            } else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
                foundMillis = true;
                assertArrayEquals(item.getValue().get(), parseLongBytes("555"));
            }
        }
    }
    assertTrue(foundDegrees & foundMillis & foundName & foundSid);
}

Example 15 with SortedMap

use of java.util.SortedMap in project hive by apache.

the class TestHiveAccumuloTableInputFormat method testDegreesAndMillis.

@Test
public void testDegreesAndMillis() throws Exception {
    Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
    IteratorSetting is = new IteratorSetting(1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, DoubleCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, GreaterThanOrEqual.class.getName());
    is.addOption(PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseDoubleBytes("55.6"))));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:dgrs");
    scan.addScanIterator(is);
    IteratorSetting is2 = new IteratorSetting(2, PrimitiveComparisonFilter.FILTER_PREFIX + 2, PrimitiveComparisonFilter.class);
    is2.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, LongCompare.class.getName());
    is2.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, LessThan.class.getName());
    is2.addOption(PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseLongBytes("778"))));
    is2.addOption(PrimitiveComparisonFilter.COLUMN, "cf:mills");
    scan.addScanIterator(is2);
    boolean foundDennis = false;
    int totalCount = 0;
    for (Map.Entry<Key, Value> kv : scan) {
        boolean foundName = false;
        boolean foundSid = false;
        boolean foundDegrees = false;
        boolean foundMillis = false;
        SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
        for (Map.Entry<Key, Value> item : items.entrySet()) {
            SortedMap<Key, Value> nestedItems = PrimitiveComparisonFilter.decodeRow(item.getKey(), item.getValue());
            for (Map.Entry<Key, Value> nested : nestedItems.entrySet()) {
                if (nested.getKey().getRow().toString().equals("r3")) {
                    foundDennis = true;
                }
                if (nested.getKey().getColumnQualifier().equals(NAME)) {
                    foundName = true;
                } else if (nested.getKey().getColumnQualifier().equals(SID)) {
                    foundSid = true;
                } else if (nested.getKey().getColumnQualifier().equals(DEGREES)) {
                    foundDegrees = true;
                } else if (nested.getKey().getColumnQualifier().equals(MILLIS)) {
                    foundMillis = true;
                }
            }
        }
        totalCount++;
        assertTrue(foundDegrees & foundMillis & foundName & foundSid);
    }
    assertTrue(foundDennis);
    assertEquals(totalCount, 1);
}