Examples with NavigableMap java.util.NavigableMap used on opensource projects

Example 96 with NavigableMap

use of java.util.NavigableMap in project hbase by apache.

the class TestReplicationSourceManager method testLogRoll.

@Test
public void testLogRoll() throws Exception {
    long baseline = 1000;
    long time = baseline;
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    KeyValue kv = new KeyValue(r1, f1, r1);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    List<WALActionsListener> listeners = new ArrayList<>(1);
    listeners.add(replication);
    final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners, URLEncoder.encode("regionserver:60020", "UTF8"));
    final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
    manager.init();
    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("tableame"));
    htd.addFamily(new HColumnDescriptor(f1));
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getFamiliesKeys()) {
        scopes.put(fam, 0);
    }
    // Testing normal log rolling every 20
    for (long i = 1; i < 101; i++) {
        if (i > 1 && i % 20 == 0) {
            wal.rollWriter();
        }
        LOG.info(i);
        final long txid = wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
        wal.sync(txid);
    }
    // Simulate a rapid insert that's followed
    // by a report that's still not totally complete (missing last one)
    LOG.info(baseline + " and " + time);
    baseline += 101;
    time = baseline;
    LOG.info(baseline + " and " + time);
    for (int i = 0; i < 3; i++) {
        wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
    }
    wal.sync();
    int logNumber = 0;
    for (Map.Entry<String, SortedSet<String>> entry : manager.getWALs().get(slaveId).entrySet()) {
        logNumber += entry.getValue().size();
    }
    assertEquals(6, logNumber);
    wal.rollWriter();
    manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(), "1", 0, false, false);
    wal.append(hri, new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit, true);
    wal.sync();
    assertEquals(1, manager.getWALs().size());
// TODO Need a case with only 2 WALs and we only want to delete the first one
}

Example 97 with NavigableMap

use of java.util.NavigableMap in project hbase by apache.

the class TestFromClientSide method testClientPoolThreadLocal.

@Ignore("Flakey: HBASE-8989")
@Test
public void testClientPoolThreadLocal() throws IOException {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    int poolSize = Integer.MAX_VALUE;
    int numVersions = 3;
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
    conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
    final Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 3);
    final long ts = EnvironmentEdgeManager.currentTime();
    final Get get = new Get(ROW);
    get.addColumn(FAMILY, QUALIFIER);
    get.setMaxVersions();
    for (int versions = 1; versions <= numVersions; versions++) {
        Put put = new Put(ROW);
        put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
        table.put(put);
        Result result = table.get(get);
        NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(QUALIFIER);
        assertEquals("The number of versions of '" + FAMILY + ":" + QUALIFIER + " did not match " + versions + "; " + put.toString() + ", " + get.toString(), versions, navigableMap.size());
        for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
            assertTrue("The value at time " + entry.getKey() + " did not match what was put", Bytes.equals(VALUE, entry.getValue()));
        }
    }
    final Object waitLock = new Object();
    ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
    final AtomicReference<AssertionError> error = new AtomicReference<>(null);
    for (int versions = numVersions; versions < numVersions * 2; versions++) {
        final int versionsCopy = versions;
        executorService.submit(new Callable<Void>() {

            @Override
            public Void call() {
                try {
                    Put put = new Put(ROW);
                    put.addColumn(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
                    table.put(put);
                    Result result = table.get(get);
                    NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(QUALIFIER);
                    assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":" + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy, navigableMap.size());
                    for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
                        assertTrue("The value at time " + entry.getKey() + " did not match what was put", Bytes.equals(VALUE, entry.getValue()));
                    }
                    synchronized (waitLock) {
                        waitLock.wait();
                    }
                } catch (Exception e) {
                } catch (AssertionError e) {
                    // the error happens in a thread, it won't fail the test,
                    // need to pass it to the caller for proper handling.
                    error.set(e);
                    LOG.error(e);
                }
                return null;
            }
        });
    }
    synchronized (waitLock) {
        waitLock.notifyAll();
    }
    executorService.shutdownNow();
    assertNull(error.get());
}

Example 98 with NavigableMap

use of java.util.NavigableMap in project hbase by apache.

the class Result method getMap.

/**
   * Map of families to all versions of its qualifiers and values.
   * <p>
   * Returns a three level Map of the form:
   * <code>Map&amp;family,Map&lt;qualifier,Map&lt;timestamp,value&gt;&gt;&gt;</code>
   * <p>
   * Note: All other map returning methods make use of this map internally.
   * @return map from families to qualifiers to versions
   */
public NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> getMap() {
    if (this.familyMap != null) {
        return this.familyMap;
    }
    if (isEmpty()) {
        return null;
    }
    this.familyMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (Cell kv : this.cells) {
        byte[] family = CellUtil.cloneFamily(kv);
        NavigableMap<byte[], NavigableMap<Long, byte[]>> columnMap = familyMap.get(family);
        if (columnMap == null) {
            columnMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
            familyMap.put(family, columnMap);
        }
        byte[] qualifier = CellUtil.cloneQualifier(kv);
        NavigableMap<Long, byte[]> versionMap = columnMap.get(qualifier);
        if (versionMap == null) {
            versionMap = new TreeMap<>(new Comparator<Long>() {

                @Override
                public int compare(Long l1, Long l2) {
                    return l2.compareTo(l1);
                }
            });
            columnMap.put(qualifier, versionMap);
        }
        Long timestamp = kv.getTimestamp();
        byte[] value = CellUtil.cloneValue(kv);
        versionMap.put(timestamp, value);
    }
    return this.familyMap;
}

Example 99 with NavigableMap

use of java.util.NavigableMap in project hbase by apache.

the class AggregationClient method median.

/**
   * This is the client side interface/handler for calling the median method for a
   * given cf-cq combination. This method collects the necessary parameters
   * to compute the median and returns the median.
   * @param table
   * @param ci
   * @param scan
   * @return R the median
   * @throws Throwable
   */
public <R, S, P extends Message, Q extends Message, T extends Message> R median(final Table table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) throws Throwable {
    Pair<NavigableMap<byte[], List<S>>, List<S>> p = getMedianArgs(table, ci, scan);
    byte[] startRow = null;
    byte[] colFamily = scan.getFamilies()[0];
    NavigableSet<byte[]> quals = scan.getFamilyMap().get(colFamily);
    NavigableMap<byte[], List<S>> map = p.getFirst();
    S sumVal = p.getSecond().get(0);
    S sumWeights = p.getSecond().get(1);
    double halfSumVal = ci.divideForAvg(sumVal, 2L);
    double movingSumVal = 0;
    boolean weighted = false;
    if (quals.size() > 1) {
        weighted = true;
        halfSumVal = ci.divideForAvg(sumWeights, 2L);
    }
    for (Map.Entry<byte[], List<S>> entry : map.entrySet()) {
        S s = weighted ? entry.getValue().get(1) : entry.getValue().get(0);
        double newSumVal = movingSumVal + ci.divideForAvg(s, 1L);
        // we found the region with the median
        if (newSumVal > halfSumVal)
            break;
        movingSumVal = newSumVal;
        startRow = entry.getKey();
    }
    // scan the region with median and find it
    Scan scan2 = new Scan(scan);
    // inherit stop row from method parameter
    if (startRow != null)
        scan2.setStartRow(startRow);
    ResultScanner scanner = null;
    try {
        int cacheSize = scan2.getCaching();
        if (!scan2.getCacheBlocks() || scan2.getCaching() < 2) {
            scan2.setCacheBlocks(true);
            cacheSize = 5;
            scan2.setCaching(cacheSize);
        }
        scanner = table.getScanner(scan2);
        Result[] results = null;
        byte[] qualifier = quals.pollFirst();
        // qualifier for the weight column
        byte[] weightQualifier = weighted ? quals.pollLast() : qualifier;
        R value = null;
        do {
            results = scanner.next(cacheSize);
            if (results != null && results.length > 0) {
                for (int i = 0; i < results.length; i++) {
                    Result r = results[i];
                    // retrieve weight
                    Cell kv = r.getColumnLatestCell(colFamily, weightQualifier);
                    R newValue = ci.getValue(colFamily, weightQualifier, kv);
                    S s = ci.castToReturnType(newValue);
                    double newSumVal = movingSumVal + ci.divideForAvg(s, 1L);
                    // see if we have moved past the median
                    if (newSumVal > halfSumVal) {
                        return value;
                    }
                    movingSumVal = newSumVal;
                    kv = r.getColumnLatestCell(colFamily, qualifier);
                    value = ci.getValue(colFamily, qualifier, kv);
                }
            }
        } while (results != null && results.length > 0);
    } finally {
        if (scanner != null) {
            scanner.close();
        }
    }
    return null;
}

Example 100 with NavigableMap

use of java.util.NavigableMap in project hbase by apache.

the class AsyncAggregationClient method sumByRegion.

// the map key is the startRow of the region
private static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<NavigableMap<byte[], S>> sumByRegion(RawAsyncTable table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) {
    CompletableFuture<NavigableMap<byte[], S>> future = new CompletableFuture<NavigableMap<byte[], S>>();
    AggregateRequest req;
    try {
        req = validateArgAndGetPB(scan, ci, false);
    } catch (IOException e) {
        future.completeExceptionally(e);
        return future;
    }
    int firstPartIndex = scan.getFamilyMap().get(scan.getFamilies()[0]).size() - 1;
    AbstractAggregationCallback<NavigableMap<byte[], S>> callback = new AbstractAggregationCallback<NavigableMap<byte[], S>>(future) {

        private final NavigableMap<byte[], S> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);

        @Override
        protected void aggregate(HRegionInfo region, AggregateResponse resp) throws IOException {
            if (resp.getFirstPartCount() > 0) {
                map.put(region.getStartKey(), getPromotedValueFromProto(ci, resp, firstPartIndex));
            }
        }

        @Override
        protected NavigableMap<byte[], S> getFinalResult() {
            return map;
        }
    };
    table.coprocessorService(channel -> AggregateService.newStub(channel), (stub, controller, rpcCallback) -> stub.getMedian(controller, req, rpcCallback), scan.getStartRow(), scan.includeStartRow(), scan.getStopRow(), scan.includeStopRow(), callback);
    return future;
}