Examples with NavigableMap java.util.NavigableMap used on opensource projects

Example 86 with NavigableMap

use of java.util.NavigableMap in project metron by apache.

the class UpdateControllerIntegrationTest method test.

@Test
public void test() throws Exception {
    String guid = "bro_2";
    ResultActions result = this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0));
    try {
        result.andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.source:type").value("bro")).andExpect(jsonPath("$.guid").value(guid)).andExpect(jsonPath("$.project").doesNotExist()).andExpect(jsonPath("$.timestamp").value(2));
    } catch (Throwable t) {
        System.err.println(result.andReturn().getResponse().getContentAsString());
        throw t;
    }
    MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(TABLE);
    Assert.assertEquals(0, table.size());
    this.mockMvc.perform(patch(updateUrl + "/patch").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(patch)).andExpect(status().isOk());
    this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.source:type").value("bro")).andExpect(jsonPath("$.guid").value(guid)).andExpect(jsonPath("$.project").value("metron")).andExpect(jsonPath("$.timestamp").value(2));
    Assert.assertEquals(1, table.size());
    {
        // ensure hbase is up to date
        Get g = new Get(new HBaseDao.Key(guid, "bro").toBytes());
        Result r = table.get(g);
        NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
        Assert.assertEquals(1, columns.size());
    }
    this.mockMvc.perform(post(updateUrl + "/replace").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(replace)).andExpect(status().isOk());
    this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)).andExpect(status().isOk()).andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))).andExpect(jsonPath("$.source:type").value("bro")).andExpect(jsonPath("$.guid").value(guid)).andExpect(jsonPath("$.project").doesNotExist()).andExpect(jsonPath("$.timestamp").value(200));
    Assert.assertEquals(1, table.size());
    {
        // ensure hbase is up to date
        Get g = new Get(new HBaseDao.Key(guid, "bro").toBytes());
        Result r = table.get(g);
        NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
        Assert.assertEquals(2, columns.size());
    }
}

Example 87 with NavigableMap

use of java.util.NavigableMap in project hive by apache.

the class HiveRelDecorrelator method decorrelateRel.

public Frame decorrelateRel(HiveAggregate rel) throws SemanticException {
    if (rel.getGroupType() != Aggregate.Group.SIMPLE) {
        throw new AssertionError(Bug.CALCITE_461_FIXED);
    }
    // Aggregate itself should not reference cor vars.
    assert !cm.mapRefRelToCorRef.containsKey(rel);
    final RelNode oldInput = rel.getInput();
    final Frame frame = getInvoke(oldInput, rel);
    if (frame == null) {
        // If input has not been rewritten, do not rewrite this rel.
        return null;
    }
    // assert !frame.corVarOutputPos.isEmpty();
    final RelNode newInput = frame.r;
    // map from newInput
    Map<Integer, Integer> mapNewInputToProjOutputs = new HashMap<>();
    final int oldGroupKeyCount = rel.getGroupSet().cardinality();
    // Project projects the original expressions,
    // plus any correlated variables the input wants to pass along.
    final List<Pair<RexNode, String>> projects = Lists.newArrayList();
    List<RelDataTypeField> newInputOutput = newInput.getRowType().getFieldList();
    int newPos = 0;
    // oldInput has the original group by keys in the front.
    final NavigableMap<Integer, RexLiteral> omittedConstants = new TreeMap<>();
    for (int i = 0; i < oldGroupKeyCount; i++) {
        final RexLiteral constant = projectedLiteral(newInput, i);
        if (constant != null) {
            // Exclude constants. Aggregate({true}) occurs because Aggregate({})
            // would generate 1 row even when applied to an empty table.
            omittedConstants.put(i, constant);
            continue;
        }
        int newInputPos = frame.oldToNewOutputs.get(i);
        projects.add(RexInputRef.of2(newInputPos, newInputOutput));
        mapNewInputToProjOutputs.put(newInputPos, newPos);
        newPos++;
    }
    final SortedMap<CorDef, Integer> corDefOutputs = new TreeMap<>();
    if (!frame.corDefOutputs.isEmpty()) {
        // position oldGroupKeyCount.
        for (Map.Entry<CorDef, Integer> entry : frame.corDefOutputs.entrySet()) {
            projects.add(RexInputRef.of2(entry.getValue(), newInputOutput));
            corDefOutputs.put(entry.getKey(), newPos);
            mapNewInputToProjOutputs.put(entry.getValue(), newPos);
            newPos++;
        }
    }
    // add the remaining fields
    final int newGroupKeyCount = newPos;
    for (int i = 0; i < newInputOutput.size(); i++) {
        if (!mapNewInputToProjOutputs.containsKey(i)) {
            projects.add(RexInputRef.of2(i, newInputOutput));
            mapNewInputToProjOutputs.put(i, newPos);
            newPos++;
        }
    }
    assert newPos == newInputOutput.size();
    // This Project will be what the old input maps to,
    // replacing any previous mapping from old input).
    RelNode newProject = HiveProject.create(newInput, Pair.left(projects), Pair.right(projects));
    // update mappings:
    // oldInput ----> newInput
    // 
    // newProject
    // |
    // oldInput ----> newInput
    // 
    // is transformed to
    // 
    // oldInput ----> newProject
    // |
    // newInput
    Map<Integer, Integer> combinedMap = Maps.newHashMap();
    for (Integer oldInputPos : frame.oldToNewOutputs.keySet()) {
        combinedMap.put(oldInputPos, mapNewInputToProjOutputs.get(frame.oldToNewOutputs.get(oldInputPos)));
    }
    register(oldInput, newProject, combinedMap, corDefOutputs);
    // now it's time to rewrite the Aggregate
    final ImmutableBitSet newGroupSet = ImmutableBitSet.range(newGroupKeyCount);
    List<AggregateCall> newAggCalls = Lists.newArrayList();
    List<AggregateCall> oldAggCalls = rel.getAggCallList();
    int oldInputOutputFieldCount = rel.getGroupSet().cardinality();
    int newInputOutputFieldCount = newGroupSet.cardinality();
    int i = -1;
    for (AggregateCall oldAggCall : oldAggCalls) {
        ++i;
        List<Integer> oldAggArgs = oldAggCall.getArgList();
        List<Integer> aggArgs = Lists.newArrayList();
        // for the argument.
        for (int oldPos : oldAggArgs) {
            aggArgs.add(combinedMap.get(oldPos));
        }
        final int filterArg = oldAggCall.filterArg < 0 ? oldAggCall.filterArg : combinedMap.get(oldAggCall.filterArg);
        newAggCalls.add(oldAggCall.adaptTo(newProject, aggArgs, filterArg, oldGroupKeyCount, newGroupKeyCount));
        // The old to new output position mapping will be the same as that
        // of newProject, plus any aggregates that the oldAgg produces.
        combinedMap.put(oldInputOutputFieldCount + i, newInputOutputFieldCount + i);
    }
    relBuilder.push(new HiveAggregate(rel.getCluster(), rel.getTraitSet(), newProject, newGroupSet, null, newAggCalls));
    if (!omittedConstants.isEmpty()) {
        final List<RexNode> postProjects = new ArrayList<>(relBuilder.fields());
        for (Map.Entry<Integer, RexLiteral> entry : omittedConstants.descendingMap().entrySet()) {
            postProjects.add(entry.getKey() + frame.corDefOutputs.size(), entry.getValue());
        }
        relBuilder.project(postProjects);
    }
    // located at the same position as the input newProject.
    return register(rel, relBuilder.build(), combinedMap, corDefOutputs);
}

Example 88 with NavigableMap

use of java.util.NavigableMap in project cdap by caskdata.

the class IncrementHandler method prePut.

@Override
public void prePut(ObserverContext<RegionCoprocessorEnvironment> ctx, Put put, WALEdit edit, Durability durability) throws IOException {
    // we assume that if any of the column families written to are transactional, the entire write is transactional
    boolean transactional = state.containsTransactionalFamily(put.getFamilyCellMap().keySet());
    boolean isIncrement = put.getAttribute(HBaseTable.DELTA_WRITE) != null;
    if (isIncrement || !transactional) {
        // incremental write
        NavigableMap<byte[], List<Cell>> newFamilyMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        long tsToAssign = 0;
        if (!transactional) {
            tsToAssign = state.getUniqueTimestamp();
        }
        for (Map.Entry<byte[], List<Cell>> entry : put.getFamilyCellMap().entrySet()) {
            List<Cell> newCells = new ArrayList<>(entry.getValue().size());
            for (Cell cell : entry.getValue()) {
                // rewrite the cell value with a special prefix to identify it as a delta
                // for 0.98 we can update this to use cell tags
                byte[] newValue = isIncrement ? Bytes.add(IncrementHandlerState.DELTA_MAGIC_PREFIX, CellUtil.cloneValue(cell)) : CellUtil.cloneValue(cell);
                newCells.add(CellUtil.createCell(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), transactional ? cell.getTimestamp() : tsToAssign, cell.getTypeByte(), newValue));
            }
            newFamilyMap.put(entry.getKey(), newCells);
        }
        put.setFamilyCellMap(newFamilyMap);
    }
// put completes normally with value prefix marker
}

Example 89 with NavigableMap

use of java.util.NavigableMap in project cdap by caskdata.

the class ConsumerConfigCache method updateCache.

/**
   * This forces an immediate update of the config cache. It should only be called from the refresh thread or from
   * tests, to avoid having to add a sleep for the duration of the refresh interval.
   *
   * This method is synchronized to protect from race conditions if called directly from a test. Otherwise this is
   * only called from the refresh thread, and there will not be concurrent invocations.
   *
   * @throws IOException if failed to update config cache
   */
@VisibleForTesting
public synchronized void updateCache() throws IOException {
    Map<byte[], QueueConsumerConfig> newCache = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
    long now = System.currentTimeMillis();
    TransactionVisibilityState txSnapshot = transactionSnapshotSupplier.get();
    if (txSnapshot == null) {
        LOG.debug("No transaction snapshot is available. Not updating the consumer config cache.");
        return;
    }
    HTableInterface table = hTableSupplier.getInput();
    try {
        // Scan the table with the transaction snapshot
        Scan scan = new Scan();
        scan.addFamily(QueueEntryRow.COLUMN_FAMILY);
        Transaction tx = TxUtils.createDummyTransaction(txSnapshot);
        setScanAttribute(scan, TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txCodec.encode(tx));
        ResultScanner scanner = table.getScanner(scan);
        int configCnt = 0;
        for (Result result : scanner) {
            if (!result.isEmpty()) {
                NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(QueueEntryRow.COLUMN_FAMILY);
                if (familyMap != null) {
                    configCnt++;
                    Map<ConsumerInstance, byte[]> consumerInstances = new HashMap<>();
                    // Gather the startRow of all instances across all consumer groups.
                    int numGroups = 0;
                    Long groupId = null;
                    for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
                        if (entry.getKey().length != STATE_COLUMN_SIZE) {
                            continue;
                        }
                        long gid = Bytes.toLong(entry.getKey());
                        int instanceId = Bytes.toInt(entry.getKey(), Bytes.SIZEOF_LONG);
                        consumerInstances.put(new ConsumerInstance(gid, instanceId), entry.getValue());
                        // Columns are sorted by groupId, hence if it change, then numGroups would get +1
                        if (groupId == null || groupId != gid) {
                            numGroups++;
                            groupId = gid;
                        }
                    }
                    byte[] queueName = result.getRow();
                    newCache.put(queueName, new QueueConsumerConfig(consumerInstances, numGroups));
                }
            }
        }
        long elapsed = System.currentTimeMillis() - now;
        this.configCache = newCache;
        this.lastUpdated = now;
        if (LOG.isDebugEnabled()) {
            LOG.debug("Updated consumer config cache with {} entries, took {} msec", configCnt, elapsed);
        }
    } finally {
        try {
            table.close();
        } catch (IOException ioe) {
            LOG.error("Error closing table {}", queueConfigTableName, ioe);
        }
    }
}

Example 90 with NavigableMap

use of java.util.NavigableMap in project cdap by caskdata.

the class IncrementHandler method preIncrementAfterRowLock.

@Override
public Result preIncrementAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> e, Increment increment) throws IOException {
    // this should only trigger for a transactional readless increment
    boolean isIncrement = increment.getAttribute(HBaseTable.DELTA_WRITE) != null;
    boolean transactional = state.containsTransactionalFamily(increment.getFamilyCellMap().keySet());
    if (!isIncrement || !transactional) {
        return null;
    }
    byte[] txBytes = increment.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
    if (txBytes == null) {
        throw new IllegalArgumentException("Attribute " + TxConstants.TX_OPERATION_ATTRIBUTE_KEY + " must be set for transactional readless increments");
    }
    byte[] wpBytes = increment.getAttribute(HBaseTable.WRITE_POINTER);
    if (wpBytes == null) {
        throw new IllegalArgumentException("Attribute " + HBaseTable.WRITE_POINTER + " must be set for transactional readless increments");
    }
    long writeVersion = Bytes.toLong(wpBytes);
    Get get = new Get(increment.getRow());
    get.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txBytes);
    for (Map.Entry<byte[], NavigableMap<byte[], Long>> entry : increment.getFamilyMapOfLongs().entrySet()) {
        byte[] family = entry.getKey();
        for (byte[] column : entry.getValue().keySet()) {
            get.addColumn(family, column);
        }
    }
    Result result = region.get(get);
    Put put = new Put(increment.getRow());
    for (Map.Entry<byte[], NavigableMap<byte[], Long>> entry : increment.getFamilyMapOfLongs().entrySet()) {
        byte[] family = entry.getKey();
        for (Map.Entry<byte[], Long> colEntry : entry.getValue().entrySet()) {
            byte[] column = colEntry.getKey();
            long value = colEntry.getValue();
            byte[] existingValue = result.getValue(family, column);
            if (existingValue != null) {
                long delta = Bytes.toLong(existingValue);
                value += delta;
            }
            put.add(new KeyValue(increment.getRow(), family, column, writeVersion, Bytes.toBytes(value)));
        }
    }
    if (!put.isEmpty()) {
        region.put(put);
    }
    e.bypass();
    return new Result();
}