Examples with GeneratedRecordComparator org.apache.flink.table.runtime.generated.GeneratedRecordComparator used on opensource projects

Example 6 with GeneratedRecordComparator

use of org.apache.flink.table.runtime.generated.GeneratedRecordComparator in project flink by apache.

the class RowTimeSortOperatorTest method testSortOnTwoFields.

@Test
public void testSortOnTwoFields() throws Exception {
    InternalTypeInfo<RowData> inputRowType = InternalTypeInfo.ofFields(new IntType(), new BigIntType(), VarCharType.STRING_TYPE, new IntType());
    // Note: RowTimeIdx must be 0 in product environment, the value is 1 here just for simplify
    // the testing
    int rowTimeIdx = 1;
    GeneratedRecordComparator gComparator = new GeneratedRecordComparator("", "", new Object[0]) {

        private static final long serialVersionUID = -6067266199060901331L;

        @Override
        public RecordComparator newInstance(ClassLoader classLoader) {
            return IntRecordComparator.INSTANCE;
        }
    };
    RowDataHarnessAssertor assertor = new RowDataHarnessAssertor(inputRowType.toRowFieldTypes());
    RowTimeSortOperator operator = createSortOperator(inputRowType, rowTimeIdx, gComparator);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    testHarness.processElement(insertRecord(3, 3L, "Hello world", 3));
    testHarness.processElement(insertRecord(2, 2L, "Hello", 2));
    testHarness.processElement(insertRecord(6, 2L, "Luke Skywalker", 6));
    testHarness.processElement(insertRecord(5, 3L, "I am fine.", 5));
    testHarness.processElement(insertRecord(7, 1L, "Comment#1", 7));
    testHarness.processElement(insertRecord(9, 4L, "Comment#3", 9));
    testHarness.processElement(insertRecord(10, 4L, "Comment#4", 10));
    testHarness.processElement(insertRecord(8, 4L, "Comment#2", 8));
    testHarness.processElement(insertRecord(1, 1L, "Hi", 2));
    testHarness.processElement(insertRecord(1, 1L, "Hi", 1));
    testHarness.processElement(insertRecord(4, 3L, "Helloworld, how are you?", 4));
    testHarness.processElement(insertRecord(4, 5L, "Hello, how are you?", 4));
    testHarness.processWatermark(new Watermark(4L));
    List<Object> expectedOutput = new ArrayList<>();
    expectedOutput.add(insertRecord(1, 1L, "Hi", 2));
    expectedOutput.add(insertRecord(1, 1L, "Hi", 1));
    expectedOutput.add(insertRecord(7, 1L, "Comment#1", 7));
    expectedOutput.add(insertRecord(2, 2L, "Hello", 2));
    expectedOutput.add(insertRecord(6, 2L, "Luke Skywalker", 6));
    expectedOutput.add(insertRecord(3, 3L, "Hello world", 3));
    expectedOutput.add(insertRecord(4, 3L, "Helloworld, how are you?", 4));
    expectedOutput.add(insertRecord(5, 3L, "I am fine.", 5));
    expectedOutput.add(insertRecord(8, 4L, "Comment#2", 8));
    expectedOutput.add(insertRecord(9, 4L, "Comment#3", 9));
    expectedOutput.add(insertRecord(10, 4L, "Comment#4", 10));
    expectedOutput.add(new Watermark(4L));
    // do a snapshot, data could be recovered from state
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    testHarness.close();
    expectedOutput.clear();
    operator = createSortOperator(inputRowType, rowTimeIdx, gComparator);
    testHarness = createTestHarness(operator);
    testHarness.initializeState(snapshot);
    testHarness.open();
    // late data will be dropped
    testHarness.processElement(insertRecord(5, 3L, "I am fine.", 6));
    testHarness.processWatermark(new Watermark(5L));
    expectedOutput.add(insertRecord(4, 5L, "Hello, how are you?", 4));
    expectedOutput.add(new Watermark(5L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    // those watermark has no effect
    testHarness.processWatermark(new Watermark(11L));
    testHarness.processWatermark(new Watermark(12L));
    expectedOutput.add(new Watermark(11L));
    expectedOutput.add(new Watermark(12L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}

Example 7 with GeneratedRecordComparator

use of org.apache.flink.table.runtime.generated.GeneratedRecordComparator in project flink by apache.

the class SortCodeGeneratorTest method getSortBaseWithNulls.

public static Tuple2<NormalizedKeyComputer, RecordComparator> getSortBaseWithNulls(String namePrefix, RowType inputType, SortSpec sortSpec) {
    SortCodeGenerator generator = new SortCodeGenerator(new TableConfig(), inputType, sortSpec);
    GeneratedNormalizedKeyComputer computer = generator.generateNormalizedKeyComputer(namePrefix + "Computer");
    GeneratedRecordComparator comparator = generator.generateRecordComparator(namePrefix + "Comparator");
    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    return new Tuple2<>(computer.newInstance(cl), comparator.newInstance(cl));
}

Example 8 with GeneratedRecordComparator

use of org.apache.flink.table.runtime.generated.GeneratedRecordComparator in project flink by apache.

the class BatchExecOverAggregate method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputType = (RowType) inputEdge.getOutputType();
    // The generated sort is used for generating the comparator among partitions.
    // So here not care the ASC or DESC for the grouping fields.
    // TODO just replace comparator to equaliser
    final int[] partitionFields = overSpec.getPartition().getFieldIndices();
    final GeneratedRecordComparator genComparator = ComparatorCodeGenerator.gen(config.getTableConfig(), "SortComparator", inputType, SortUtil.getAscendingSortSpec(partitionFields));
    // use aggInputType which considers constants as input instead of inputType
    final RowType inputTypeWithConstants = getInputTypeWithConstants();
    // Over operator could support different order-by keys with collation satisfied.
    // Currently, this operator requires all order keys (combined with partition keys) are
    // the same, but order-by keys may be different. Consider the following sql:
    // select *, sum(b) over partition by a order by a, count(c) over partition by a from T
    // So we can use any one from the groups. To keep the behavior with the rule, we use the
    // last one.
    final SortSpec sortSpec = overSpec.getGroups().get(overSpec.getGroups().size() - 1).getSort();
    final TableStreamOperator<RowData> operator;
    final long managedMemory;
    if (!needBufferData()) {
        // operator needn't cache data
        final int numOfGroup = overSpec.getGroups().size();
        final GeneratedAggsHandleFunction[] aggsHandlers = new GeneratedAggsHandleFunction[numOfGroup];
        final boolean[] resetAccumulators = new boolean[numOfGroup];
        for (int i = 0; i < numOfGroup; ++i) {
            GroupSpec group = overSpec.getGroups().get(i);
            AggregateInfoList aggInfoList = AggregateUtil.transformToBatchAggregateInfoList(inputTypeWithConstants, JavaScalaConversionUtil.toScala(group.getAggCalls()), // aggCallNeedRetractions
            null, sortSpec.getFieldIndices());
            AggsHandlerCodeGenerator generator = new AggsHandlerCodeGenerator(new CodeGeneratorContext(config.getTableConfig()), planner.getRelBuilder(), JavaScalaConversionUtil.toScala(inputType.getChildren()), // copyInputField
            false);
            // over agg code gen must pass the constants
            aggsHandlers[i] = generator.needAccumulate().withConstants(JavaScalaConversionUtil.toScala(getConstants())).generateAggsHandler("BoundedOverAggregateHelper", aggInfoList);
            OverWindowMode mode = inferGroupMode(group);
            resetAccumulators[i] = mode == OverWindowMode.ROW && group.getLowerBound().isCurrentRow() && group.getUpperBound().isCurrentRow();
        }
        operator = new NonBufferOverWindowOperator(aggsHandlers, genComparator, resetAccumulators);
        managedMemory = 0L;
    } else {
        List<OverWindowFrame> windowFrames = createOverWindowFrames(planner.getRelBuilder(), config, inputType, sortSpec, inputTypeWithConstants);
        operator = new BufferDataOverWindowOperator(windowFrames.toArray(new OverWindowFrame[0]), genComparator, inputType.getChildren().stream().allMatch(BinaryRowData::isInFixedLengthPart));
        managedMemory = config.get(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_EXTERNAL_BUFFER_MEMORY).getBytes();
    }
    return ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationName(config), createTransformationDescription(config), SimpleOperatorFactory.of(operator), InternalTypeInfo.of(getOutputType()), inputTransform.getParallelism(), managedMemory);
}

Example 9 with GeneratedRecordComparator

use of org.apache.flink.table.runtime.generated.GeneratedRecordComparator in project flink by apache.

the class StreamExecRank method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    switch(rankType) {
        case ROW_NUMBER:
            break;
        case RANK:
            throw new TableException("RANK() on streaming table is not supported currently");
        case DENSE_RANK:
            throw new TableException("DENSE_RANK() on streaming table is not supported currently");
        default:
            throw new TableException(String.format("Streaming tables do not support %s rank function.", rankType));
    }
    ExecEdge inputEdge = getInputEdges().get(0);
    Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    RowType inputType = (RowType) inputEdge.getOutputType();
    InternalTypeInfo<RowData> inputRowTypeInfo = InternalTypeInfo.of(inputType);
    int[] sortFields = sortSpec.getFieldIndices();
    RowDataKeySelector sortKeySelector = KeySelectorUtil.getRowDataSelector(sortFields, inputRowTypeInfo);
    // create a sort spec on sort keys.
    int[] sortKeyPositions = IntStream.range(0, sortFields.length).toArray();
    SortSpec.SortSpecBuilder builder = SortSpec.builder();
    IntStream.range(0, sortFields.length).forEach(idx -> builder.addField(idx, sortSpec.getFieldSpec(idx).getIsAscendingOrder(), sortSpec.getFieldSpec(idx).getNullIsLast()));
    SortSpec sortSpecInSortKey = builder.build();
    GeneratedRecordComparator sortKeyComparator = ComparatorCodeGenerator.gen(config.getTableConfig(), "StreamExecSortComparator", RowType.of(sortSpec.getFieldTypes(inputType)), sortSpecInSortKey);
    long cacheSize = config.get(TABLE_EXEC_RANK_TOPN_CACHE_SIZE);
    StateTtlConfig ttlConfig = StateConfigUtil.createTtlConfig(config.getStateRetentionTime());
    AbstractTopNFunction processFunction;
    if (rankStrategy instanceof RankProcessStrategy.AppendFastStrategy) {
        if (sortFields.length == 1 && TypeCheckUtils.isProcTime(inputType.getChildren().get(sortFields[0])) && sortSpec.getFieldSpec(0).getIsAscendingOrder()) {
            processFunction = new AppendOnlyFirstNFunction(ttlConfig, inputRowTypeInfo, sortKeyComparator, sortKeySelector, rankType, rankRange, generateUpdateBefore, outputRankNumber);
        } else if (RankUtil.isTop1(rankRange)) {
            processFunction = new FastTop1Function(ttlConfig, inputRowTypeInfo, sortKeyComparator, sortKeySelector, rankType, rankRange, generateUpdateBefore, outputRankNumber, cacheSize);
        } else {
            processFunction = new AppendOnlyTopNFunction(ttlConfig, inputRowTypeInfo, sortKeyComparator, sortKeySelector, rankType, rankRange, generateUpdateBefore, outputRankNumber, cacheSize);
        }
    } else if (rankStrategy instanceof RankProcessStrategy.UpdateFastStrategy) {
        if (RankUtil.isTop1(rankRange)) {
            processFunction = new FastTop1Function(ttlConfig, inputRowTypeInfo, sortKeyComparator, sortKeySelector, rankType, rankRange, generateUpdateBefore, outputRankNumber, cacheSize);
        } else {
            RankProcessStrategy.UpdateFastStrategy updateFastStrategy = (RankProcessStrategy.UpdateFastStrategy) rankStrategy;
            int[] primaryKeys = updateFastStrategy.getPrimaryKeys();
            RowDataKeySelector rowKeySelector = KeySelectorUtil.getRowDataSelector(primaryKeys, inputRowTypeInfo);
            processFunction = new UpdatableTopNFunction(ttlConfig, inputRowTypeInfo, rowKeySelector, sortKeyComparator, sortKeySelector, rankType, rankRange, generateUpdateBefore, outputRankNumber, cacheSize);
        }
    // TODO Use UnaryUpdateTopNFunction after SortedMapState is merged
    } else if (rankStrategy instanceof RankProcessStrategy.RetractStrategy) {
        EqualiserCodeGenerator equaliserCodeGen = new EqualiserCodeGenerator(inputType.getFields().stream().map(RowType.RowField::getType).toArray(LogicalType[]::new));
        GeneratedRecordEqualiser generatedEqualiser = equaliserCodeGen.generateRecordEqualiser("RankValueEqualiser");
        ComparableRecordComparator comparator = new ComparableRecordComparator(sortKeyComparator, sortKeyPositions, sortSpec.getFieldTypes(inputType), sortSpec.getAscendingOrders(), sortSpec.getNullsIsLast());
        processFunction = new RetractableTopNFunction(ttlConfig, inputRowTypeInfo, comparator, sortKeySelector, rankType, rankRange, generatedEqualiser, generateUpdateBefore, outputRankNumber);
    } else {
        throw new TableException(String.format("rank strategy:%s is not supported.", rankStrategy));
    }
    KeyedProcessOperator<RowData, RowData, RowData> operator = new KeyedProcessOperator<>(processFunction);
    processFunction.setKeyContext(operator);
    OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(RANK_TRANSFORMATION, config), operator, InternalTypeInfo.of((RowType) getOutputType()), inputTransform.getParallelism());
    // set KeyType and Selector for state
    RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(partitionSpec.getFieldIndices(), inputRowTypeInfo);
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    return transform;
}

Example 10 with GeneratedRecordComparator

use of org.apache.flink.table.runtime.generated.GeneratedRecordComparator in project flink by apache.

the class StreamExecWindowRank method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    // validate rank type
    switch(rankType) {
        case ROW_NUMBER:
            break;
        case RANK:
            throw new TableException("RANK() function is not supported on Window TopN currently, only ROW_NUMBER() is supported.");
        case DENSE_RANK:
            throw new TableException("DENSE_RANK() function is not supported on Window TopN currently, only ROW_NUMBER() is supported.");
        default:
            throw new TableException(String.format("%s() function is not supported on Window TopN currently, only ROW_NUMBER() is supported.", rankType));
    }
    // validate window strategy
    if (!windowing.isRowtime()) {
        throw new TableException("Processing time Window TopN is not supported yet.");
    }
    int windowEndIndex;
    if (windowing instanceof WindowAttachedWindowingStrategy) {
        windowEndIndex = ((WindowAttachedWindowingStrategy) windowing).getWindowEnd();
    } else {
        throw new UnsupportedOperationException(windowing.getClass().getName() + " is not supported yet.");
    }
    ExecEdge inputEdge = getInputEdges().get(0);
    RowType inputType = (RowType) inputEdge.getOutputType();
    // validate rank range
    ConstantRankRange constantRankRange;
    if (rankRange instanceof ConstantRankRange) {
        constantRankRange = (ConstantRankRange) rankRange;
    } else {
        throw new TableException(String.format("Rank strategy %s is not supported on window rank currently.", rankRange.toString(inputType.getFieldNames())));
    }
    Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    InternalTypeInfo<RowData> inputRowTypeInfo = InternalTypeInfo.of(inputType);
    int[] sortFields = sortSpec.getFieldIndices();
    RowDataKeySelector sortKeySelector = KeySelectorUtil.getRowDataSelector(sortFields, inputRowTypeInfo);
    SortSpec.SortSpecBuilder builder = SortSpec.builder();
    IntStream.range(0, sortFields.length).forEach(idx -> builder.addField(idx, sortSpec.getFieldSpec(idx).getIsAscendingOrder(), sortSpec.getFieldSpec(idx).getNullIsLast()));
    SortSpec sortSpecInSortKey = builder.build();
    ZoneId shiftTimeZone = TimeWindowUtil.getShiftTimeZone(windowing.getTimeAttributeType(), config.getLocalTimeZone());
    GeneratedRecordComparator sortKeyComparator = ComparatorCodeGenerator.gen(config.getTableConfig(), "StreamExecSortComparator", RowType.of(sortSpec.getFieldTypes(inputType)), sortSpecInSortKey);
    RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(partitionSpec.getFieldIndices(), inputRowTypeInfo);
    OneInputStreamOperator<RowData, RowData> operator = WindowRankOperatorBuilder.builder().inputSerializer(new RowDataSerializer(inputType)).shiftTimeZone(shiftTimeZone).keySerializer((PagedTypeSerializer<RowData>) selector.getProducedType().toSerializer()).sortKeySelector(sortKeySelector).sortKeyComparator(sortKeyComparator).outputRankNumber(outputRankNumber).rankStart(constantRankRange.getRankStart()).rankEnd(constantRankRange.getRankEnd()).windowEndIndex(windowEndIndex).build();
    OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(WINDOW_RANK_TRANSFORMATION, config), SimpleOperatorFactory.of(operator), InternalTypeInfo.of(getOutputType()), inputTransform.getParallelism(), WINDOW_RANK_MEMORY_RATIO);
    // set KeyType and Selector for state
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    return transform;
}