Examples with ImmutableList com.google.common.collect.ImmutableList used on opensource projects

Example 1 with ImmutableList

use of com.google.common.collect.ImmutableList in project hbase by apache.

the class StripeCompactionPolicy method selectSingleStripeCompaction.

protected StripeCompactionRequest selectSingleStripeCompaction(StripeInformationProvider si, boolean includeL0, boolean canDropDeletesWithoutL0, boolean isOffpeak) throws IOException {
    ArrayList<ImmutableList<StoreFile>> stripes = si.getStripes();
    int bqIndex = -1;
    List<StoreFile> bqSelection = null;
    int stripeCount = stripes.size();
    long bqTotalSize = -1;
    for (int i = 0; i < stripeCount; ++i) {
        // If we want to compact L0 to drop deletes, we only want whole-stripe compactions.
        // So, pass includeL0 as 2nd parameter to indicate that.
        List<StoreFile> selection = selectSimpleCompaction(stripes.get(i), !canDropDeletesWithoutL0 && includeL0, isOffpeak);
        if (selection.isEmpty())
            continue;
        long size = 0;
        for (StoreFile sf : selection) {
            size += sf.getReader().length();
        }
        if (bqSelection == null || selection.size() > bqSelection.size() || (selection.size() == bqSelection.size() && size < bqTotalSize)) {
            bqSelection = selection;
            bqIndex = i;
            bqTotalSize = size;
        }
    }
    if (bqSelection == null) {
        LOG.debug("No good compaction is possible in any stripe");
        return null;
    }
    List<StoreFile> filesToCompact = new ArrayList<>(bqSelection);
    // See if we can, and need to, split this stripe.
    int targetCount = 1;
    long targetKvs = Long.MAX_VALUE;
    boolean hasAllFiles = filesToCompact.size() == stripes.get(bqIndex).size();
    String splitString = "";
    if (hasAllFiles && bqTotalSize >= config.getSplitSize()) {
        if (includeL0) {
            // So, if we might split, don't compact the stripe with L0.
            return null;
        }
        Pair<Long, Integer> kvsAndCount = estimateTargetKvs(filesToCompact, config.getSplitCount());
        targetKvs = kvsAndCount.getFirst();
        targetCount = kvsAndCount.getSecond();
        splitString = "; the stripe will be split into at most " + targetCount + " stripes with " + targetKvs + " target KVs";
    }
    LOG.debug("Found compaction in a stripe with end key [" + Bytes.toString(si.getEndRow(bqIndex)) + "], with " + filesToCompact.size() + " files of total size " + bqTotalSize + splitString);
    // See if we can drop deletes.
    StripeCompactionRequest req;
    if (includeL0) {
        assert hasAllFiles;
        List<StoreFile> l0Files = si.getLevel0Files();
        LOG.debug("Adding " + l0Files.size() + " files to compaction to be able to drop deletes");
        ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<>();
        sfs.addSublist(filesToCompact);
        sfs.addSublist(l0Files);
        req = new BoundaryStripeCompactionRequest(sfs, si.getStripeBoundaries());
    } else {
        req = new SplitStripeCompactionRequest(filesToCompact, si.getStartRow(bqIndex), si.getEndRow(bqIndex), targetCount, targetKvs);
    }
    if (hasAllFiles && (canDropDeletesWithoutL0 || includeL0)) {
        req.setMajorRange(si.getStartRow(bqIndex), si.getEndRow(bqIndex));
    }
    req.getRequest().setOffPeak(isOffpeak);
    return req;
}

Example 2 with ImmutableList

use of com.google.common.collect.ImmutableList in project hive by apache.

the class HiveSubQRemoveRelBuilder method values.

/** Creates a {@link Values}.
   *
   * <p>The {@code values} array must have the same number of entries as
   * {@code fieldNames}, or an integer multiple if you wish to create multiple
   * rows.
   *
   * <p>If there are zero rows, or if all values of a any column are
   * null, this method cannot deduce the type of columns. For these cases,
   * call {@link #values(Iterable, RelDataType)}.
   *
   * @param fieldNames Field names
   * @param values Values
   */
public HiveSubQRemoveRelBuilder values(String[] fieldNames, Object... values) {
    if (fieldNames == null || fieldNames.length == 0 || values.length % fieldNames.length != 0 || values.length < fieldNames.length) {
        throw new IllegalArgumentException("Value count must be a positive multiple of field count");
    }
    final int rowCount = values.length / fieldNames.length;
    for (Ord<String> fieldName : Ord.zip(fieldNames)) {
        if (allNull(values, fieldName.i, fieldNames.length)) {
            throw new IllegalArgumentException("All values of field '" + fieldName.e + "' are null; cannot deduce type");
        }
    }
    final ImmutableList<ImmutableList<RexLiteral>> tupleList = tupleList(fieldNames.length, values);
    final RelDataTypeFactory.FieldInfoBuilder rowTypeBuilder = cluster.getTypeFactory().builder();
    for (final Ord<String> fieldName : Ord.zip(fieldNames)) {
        final String name = fieldName.e != null ? fieldName.e : "expr$" + fieldName.i;
        final RelDataType type = cluster.getTypeFactory().leastRestrictive(new AbstractList<RelDataType>() {

            public RelDataType get(int index) {
                return tupleList.get(index).get(fieldName.i).getType();
            }

            public int size() {
                return rowCount;
            }
        });
        rowTypeBuilder.add(name, type);
    }
    final RelDataType rowType = rowTypeBuilder.build();
    return values(tupleList, rowType);
}

Example 3 with ImmutableList

use of com.google.common.collect.ImmutableList in project hive by apache.

the class HiveCalciteUtil method getPredsNotPushedAlready.

/**
   * Given a list of predicates to push down, this methods returns the set of predicates
   * that still need to be pushed. Predicates need to be pushed because 1) their String
   * representation is not included in input set of predicates to exclude, or 2) they are
   * already in the subtree rooted at the input node.
   * This method updates the set of predicates to exclude with the String representation
   * of the predicates in the output and in the subtree.
   *
   * @param predicatesToExclude String representation of predicates that should be excluded
   * @param inp root of the subtree
   * @param predsToPushDown candidate predicates to push down through the subtree
   * @return list of predicates to push down
   */
public static ImmutableList<RexNode> getPredsNotPushedAlready(Set<String> predicatesToExclude, RelNode inp, List<RexNode> predsToPushDown) {
    // Bail out if there is nothing to push
    if (predsToPushDown.isEmpty()) {
        return ImmutableList.of();
    }
    // Build map to not convert multiple times, further remove already included predicates
    Map<String, RexNode> stringToRexNode = Maps.newLinkedHashMap();
    for (RexNode r : predsToPushDown) {
        String rexNodeString = r.toString();
        if (predicatesToExclude.add(rexNodeString)) {
            stringToRexNode.put(rexNodeString, r);
        }
    }
    if (stringToRexNode.isEmpty()) {
        return ImmutableList.of();
    }
    // Finally exclude preds that are already in the subtree as given by the metadata provider
    // Note: this is the last step, trying to avoid the expensive call to the metadata provider
    //       if possible
    Set<String> predicatesInSubtree = Sets.newHashSet();
    for (RexNode pred : RelMetadataQuery.instance().getPulledUpPredicates(inp).pulledUpPredicates) {
        predicatesInSubtree.add(pred.toString());
        predicatesInSubtree.addAll(Lists.transform(RelOptUtil.conjunctions(pred), REX_STR_FN));
    }
    final ImmutableList.Builder<RexNode> newConjuncts = ImmutableList.builder();
    for (Entry<String, RexNode> e : stringToRexNode.entrySet()) {
        if (predicatesInSubtree.add(e.getKey())) {
            newConjuncts.add(e.getValue());
        }
    }
    predicatesToExclude.addAll(predicatesInSubtree);
    return newConjuncts.build();
}

Example 4 with ImmutableList

use of com.google.common.collect.ImmutableList in project hive by apache.

the class HiveRelMdCollation method collations.

public ImmutableList<RelCollation> collations(HiveAggregate aggregate, RelMetadataQuery mq) {
    // Compute collations
    ImmutableList.Builder<RelFieldCollation> collationListBuilder = new ImmutableList.Builder<RelFieldCollation>();
    for (int pos : aggregate.getGroupSet().asList()) {
        final RelFieldCollation fieldCollation = new RelFieldCollation(pos);
        collationListBuilder.add(fieldCollation);
    }
    // Return aggregate collations
    return ImmutableList.of(RelCollationTraitDef.INSTANCE.canonize(new HiveRelCollation(collationListBuilder.build())));
}

Example 5 with ImmutableList

use of com.google.common.collect.ImmutableList in project hbase by apache.

the class TestStripeCompactionPolicy method createStripesWithFiles.

/**
   * This method actually does all the work.
   */
private static StripeInformationProvider createStripesWithFiles(List<byte[]> boundaries, List<List<StoreFile>> stripeFiles, List<StoreFile> l0Files) throws Exception {
    ArrayList<ImmutableList<StoreFile>> stripes = new ArrayList<>();
    ArrayList<byte[]> boundariesList = new ArrayList<>();
    StripeInformationProvider si = mock(StripeInformationProvider.class);
    if (!stripeFiles.isEmpty()) {
        assert stripeFiles.size() == (boundaries.size() + 1);
        boundariesList.add(OPEN_KEY);
        for (int i = 0; i <= boundaries.size(); ++i) {
            byte[] startKey = ((i == 0) ? OPEN_KEY : boundaries.get(i - 1));
            byte[] endKey = ((i == boundaries.size()) ? OPEN_KEY : boundaries.get(i));
            boundariesList.add(endKey);
            for (StoreFile sf : stripeFiles.get(i)) {
                setFileStripe(sf, startKey, endKey);
            }
            stripes.add(ImmutableList.copyOf(stripeFiles.get(i)));
            when(si.getStartRow(eq(i))).thenReturn(startKey);
            when(si.getEndRow(eq(i))).thenReturn(endKey);
        }
    }
    ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<>();
    sfs.addAllSublists(stripes);
    sfs.addSublist(l0Files);
    when(si.getStorefiles()).thenReturn(sfs);
    when(si.getStripes()).thenReturn(stripes);
    when(si.getStripeBoundaries()).thenReturn(boundariesList);
    when(si.getStripeCount()).thenReturn(stripes.size());
    when(si.getLevel0Files()).thenReturn(l0Files);
    return si;
}