Examples with Pair org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair used on opensource projects

Example 51 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project hive by apache.

the class RelOptHiveTable method generateKeys.

private Pair<List<ImmutableBitSet>, List<ImmutableBitSet>> generateKeys() {
    final PrimaryKeyInfo primaryKeyInfo = hiveTblMetadata.getPrimaryKeyInfo();
    final UniqueConstraint uniqueKeyInfo = hiveTblMetadata.getUniqueKeyInfo();
    ImmutableList.Builder<ImmutableBitSet> builder = ImmutableList.builder();
    ImmutableList.Builder<ImmutableBitSet> nonNullbuilder = ImmutableList.builder();
    // First PK
    if (primaryKeyInfo != null && !primaryKeyInfo.getColNames().isEmpty()) {
        ImmutableBitSet.Builder keys = ImmutableBitSet.builder();
        for (String pkColName : primaryKeyInfo.getColNames().values()) {
            int pkPos;
            for (pkPos = 0; pkPos < rowType.getFieldNames().size(); pkPos++) {
                String colName = rowType.getFieldNames().get(pkPos);
                if (pkColName.equals(colName)) {
                    break;
                }
            }
            if (pkPos == rowType.getFieldNames().size()) {
                LOG.error("Column for primary key definition " + pkColName + " not found");
            }
            keys.set(pkPos);
        }
        ImmutableBitSet key = keys.build();
        builder.add(key);
        nonNullbuilder.add(key);
    }
    // Then UKs
    if (uniqueKeyInfo != null && !uniqueKeyInfo.getUniqueConstraints().isEmpty()) {
        for (List<UniqueConstraintCol> ukCols : uniqueKeyInfo.getUniqueConstraints().values()) {
            ImmutableBitSet.Builder keys = ImmutableBitSet.builder();
            boolean isNonNullable = true;
            for (UniqueConstraintCol ukCol : ukCols) {
                int ukPos;
                for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) {
                    String colName = rowType.getFieldNames().get(ukPos);
                    if (ukCol.colName.equals(colName)) {
                        if (rowType.getFieldList().get(ukPos).getType().isNullable()) {
                            // they should all be nullable
                            isNonNullable = false;
                        }
                        break;
                    }
                }
                if (ukPos == rowType.getFieldNames().size()) {
                    LOG.error("Column for unique constraint definition " + ukCol.colName + " not found");
                }
                keys.set(ukPos);
            }
            ImmutableBitSet key = keys.build();
            builder.add(key);
            if (isNonNullable) {
                nonNullbuilder.add(key);
            }
        }
    }
    return new Pair<>(builder.build(), nonNullbuilder.build());
}

Example 52 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project hive by apache.

the class ParallelEdgeFixer method fixParallelEdges.

private void fixParallelEdges(OperatorGraph og) throws SemanticException {
    // Identify edge operators
    ListValuedMap<Pair<Cluster, Cluster>, Pair<Operator<?>, Operator<?>>> edgeOperators = new ArrayListValuedHashMap<>();
    for (Cluster c : og.getClusters()) {
        for (Operator<?> o : c.getMembers()) {
            for (Operator<? extends OperatorDesc> p : o.getParentOperators()) {
                Cluster parentCluster = og.clusterOf(p);
                if (parentCluster == c) {
                    continue;
                }
                edgeOperators.put(new Pair<>(parentCluster, c), new Pair<>(p, o));
            }
        }
    }
    // process all edges and fix parallel edges if there are any
    for (Pair<Cluster, Cluster> key : edgeOperators.keySet()) {
        List<Pair<Operator<?>, Operator<?>>> values = edgeOperators.get(key);
        if (values.size() <= 1) {
            continue;
        }
        // operator order must in stabile order - or we end up with falky plans causing flaky tests...
        values.sort(new OperatorPairComparator());
        // remove one optionally unsupported edge (it will be kept as is)
        removeOneEdge(values);
        Iterator<Pair<Operator<?>, Operator<?>>> it = values.iterator();
        while (it.hasNext()) {
            Pair<Operator<?>, Operator<?>> pair = it.next();
            fixParallelEdge(pair.left, pair.right);
        }
    }
}

Example 53 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project hive by apache.

the class HiveProjectVisitor method createColInfos.

private Pair<ArrayList<ColumnInfo>, Set<Integer>> createColInfos(List<RexNode> calciteExprs, List<ExprNodeDesc> hiveExprs, List<String> projNames, OpAttr inpOpAf) {
    if (hiveExprs.size() != projNames.size()) {
        throw new RuntimeException("Column expressions list doesn't match Column Names list");
    }
    RexNode rexN;
    ExprNodeDesc pe;
    ArrayList<ColumnInfo> colInfos = new ArrayList<ColumnInfo>();
    boolean vc;
    Set<Integer> newVColSet = new HashSet<Integer>();
    for (int i = 0; i < hiveExprs.size(); i++) {
        pe = hiveExprs.get(i);
        rexN = calciteExprs.get(i);
        vc = false;
        if (rexN instanceof RexInputRef) {
            if (inpOpAf.vcolsInCalcite.contains(((RexInputRef) rexN).getIndex())) {
                newVColSet.add(i);
                vc = true;
            }
        }
        colInfos.add(new ColumnInfo(projNames.get(i), pe.getTypeInfo(), inpOpAf.tabAlias, vc));
    }
    return new Pair<ArrayList<ColumnInfo>, Set<Integer>>(colInfos, newVColSet);
}

Example 54 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project hive by apache.

the class HiveRelMdRowCount method canHandleJoin.

/*
   * 1. Join condition must be an Equality Predicate.
   * 2. both sides must reference 1 column.
   * 3. If needed flip the columns.
   */
private static Pair<Integer, Integer> canHandleJoin(Join joinRel, List<RexNode> leftFilters, List<RexNode> rightFilters, List<RexNode> joinFilters) {
    /*
     * If after classifying filters there is more than 1 joining predicate, we
     * don't handle this. Return null.
     */
    if (joinFilters.size() != 1) {
        return null;
    }
    RexNode joinCond = joinFilters.get(0);
    int leftColIdx;
    int rightColIdx;
    if (!(joinCond instanceof RexCall)) {
        return null;
    }
    if (((RexCall) joinCond).getOperator() != SqlStdOperatorTable.EQUALS) {
        return null;
    }
    ImmutableBitSet leftCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(0));
    ImmutableBitSet rightCols = RelOptUtil.InputFinder.bits(((RexCall) joinCond).getOperands().get(1));
    if (leftCols.cardinality() != 1 || rightCols.cardinality() != 1) {
        return null;
    }
    int nFieldsLeft = joinRel.getLeft().getRowType().getFieldList().size();
    int nFieldsRight = joinRel.getRight().getRowType().getFieldList().size();
    int nSysFields = joinRel.getSystemFieldList().size();
    ImmutableBitSet rightFieldsBitSet = ImmutableBitSet.range(nSysFields + nFieldsLeft, nSysFields + nFieldsLeft + nFieldsRight);
    /*
     * flip column references if join condition specified in reverse order to
     * join sources.
     */
    if (rightFieldsBitSet.contains(leftCols)) {
        ImmutableBitSet t = leftCols;
        leftCols = rightCols;
        rightCols = t;
    }
    leftColIdx = leftCols.nextSetBit(0) - nSysFields;
    rightColIdx = rightCols.nextSetBit(0) - (nSysFields + nFieldsLeft);
    return new Pair<Integer, Integer>(leftColIdx, rightColIdx);
}

Example 55 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project hive by apache.

the class TestMaterializedViewsCache method testParallelism.

@Disabled("Testing parallelism only")
@Test
void testParallelism() {
    int ITERATIONS = 1000000;
    List<Pair<Table, HiveRelOptMaterialization>> testData = new ArrayList<>();
    for (int i = 0; i < 10; ++i) {
        Table table = new Table(new org.apache.hadoop.hive.metastore.api.Table());
        table.setDbName("default");
        table.setTableName("mat" + i);
        table.setViewOriginalText("select col0 from t" + i);
        HiveRelOptMaterialization materialization = createMaterialization(table);
        testData.add(new Pair<>(table, materialization));
    }
    for (int i = 0; i < 10; ++i) {
        Table table = new Table(new org.apache.hadoop.hive.metastore.api.Table());
        table.setDbName("db1");
        table.setTableName("mat" + i);
        table.setViewOriginalText("select col0 from t" + i);
        HiveRelOptMaterialization materialization = createMaterialization(table);
        testData.add(new Pair<>(table, materialization));
    }
    List<Callable<Void>> callableList = new ArrayList<>();
    callableList.add(() -> {
        for (Pair<Table, HiveRelOptMaterialization> entry : testData) {
            materializedViewsCache.refresh(entry.left, entry.left, entry.right);
        }
        return null;
    });
    callableList.add(() -> {
        for (int j = 0; j < ITERATIONS; ++j) {
            for (Pair<Table, HiveRelOptMaterialization> entry : testData) {
                materializedViewsCache.remove(entry.left);
                materializedViewsCache.putIfAbsent(entry.left, entry.right);
            }
        }
        return null;
    });
    for (Pair<Table, HiveRelOptMaterialization> entry : testData) {
        callableList.add(() -> {
            for (int j = 0; j < ITERATIONS; ++j) {
                materializedViewsCache.get(entry.left.getViewExpandedText());
            }
            return null;
        });
    }
    callableList.add(() -> {
        for (int j = 0; j < ITERATIONS; ++j) {
            List<HiveRelOptMaterialization> materializations = materializedViewsCache.values();
        }
        return null;
    });
    ExecutorService executor = Executors.newFixedThreadPool(12);
    try {
        executor.invokeAll(callableList);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}