Examples with SortSpec org.apache.flink.table.planner.plan.nodes.exec.spec.SortSpec used on opensource projects

Example 1 with SortSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.SortSpec in project flink by apache.

the class StreamExecMatch method checkOrderKeys.

private void checkOrderKeys(RowType inputRowType) {
    SortSpec orderKeys = matchSpec.getOrderKeys();
    if (orderKeys.getFieldSize() == 0) {
        throw new TableException("You must specify either rowtime or proctime for order by.");
    }
    SortSpec.SortFieldSpec timeOrderField = orderKeys.getFieldSpec(0);
    int timeOrderFieldIdx = timeOrderField.getFieldIndex();
    LogicalType timeOrderFieldType = inputRowType.getTypeAt(timeOrderFieldIdx);
    // need to identify time between others order fields. Time needs to be first sort element
    if (!TypeCheckUtils.isRowTime(timeOrderFieldType) && !TypeCheckUtils.isProcTime(timeOrderFieldType)) {
        throw new TableException("You must specify either rowtime or proctime for order by as the first one.");
    }
    // time ordering needs to be ascending
    if (!orderKeys.getAscendingOrders()[0]) {
        throw new TableException("Primary sort order of a streaming table must be ascending on time.");
    }
}

Example 2 with SortSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.SortSpec in project flink by apache.

the class StreamExecTemporalSort method createSortProcTime.

/**
 * Create Sort logic based on processing time.
 */
private Transformation<RowData> createSortProcTime(RowType inputType, Transformation<RowData> inputTransform, ExecNodeConfig config) {
    // if the order has secondary sorting fields in addition to the proctime
    if (sortSpec.getFieldSize() > 1) {
        // skip the first field which is the proctime field and would be ordered by timer.
        SortSpec specExcludeTime = sortSpec.createSubSortSpec(1);
        GeneratedRecordComparator rowComparator = ComparatorCodeGenerator.gen(config.getTableConfig(), "ProcTimeSortComparator", inputType, specExcludeTime);
        ProcTimeSortOperator sortOperator = new ProcTimeSortOperator(InternalTypeInfo.of(inputType), rowComparator);
        OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(TEMPORAL_SORT_TRANSFORMATION, config), sortOperator, InternalTypeInfo.of(inputType), inputTransform.getParallelism());
        // as input node is singleton exchange, its parallelism is 1.
        if (inputsContainSingleton()) {
            transform.setParallelism(1);
            transform.setMaxParallelism(1);
        }
        EmptyRowDataKeySelector selector = EmptyRowDataKeySelector.INSTANCE;
        transform.setStateKeySelector(selector);
        transform.setStateKeyType(selector.getProducedType());
        return transform;
    } else {
        // if the order is done only on proctime we only need to forward the elements
        return inputTransform;
    }
}

Example 3 with SortSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.SortSpec in project flink by apache.

the class StreamExecTemporalSort method createSortRowTime.

/**
 * Create Sort logic based on row time.
 */
private Transformation<RowData> createSortRowTime(RowType inputType, Transformation<RowData> inputTransform, ExecNodeConfig config) {
    GeneratedRecordComparator rowComparator = null;
    if (sortSpec.getFieldSize() > 1) {
        // skip the first field which is the rowtime field and would be ordered by timer.
        SortSpec specExcludeTime = sortSpec.createSubSortSpec(1);
        rowComparator = ComparatorCodeGenerator.gen(config.getTableConfig(), "RowTimeSortComparator", inputType, specExcludeTime);
    }
    RowTimeSortOperator sortOperator = new RowTimeSortOperator(InternalTypeInfo.of(inputType), sortSpec.getFieldSpec(0).getFieldIndex(), rowComparator);
    OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(inputTransform, createTransformationMeta(TEMPORAL_SORT_TRANSFORMATION, config), sortOperator, InternalTypeInfo.of(inputType), inputTransform.getParallelism());
    if (inputsContainSingleton()) {
        transform.setParallelism(1);
        transform.setMaxParallelism(1);
    }
    EmptyRowDataKeySelector selector = EmptyRowDataKeySelector.INSTANCE;
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    return transform;
}

Example 4 with SortSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.SortSpec in project flink by apache.

the class BatchExecPythonOverAggregate method getPythonOverWindowAggregateFunctionOperator.

@SuppressWarnings("unchecked")
private OneInputStreamOperator<RowData, RowData> getPythonOverWindowAggregateFunctionOperator(ExecNodeConfig config, Configuration pythonConfig, RowType inputRowType, RowType outputRowType, boolean[] isRangeWindows, int[] udafInputOffsets, PythonFunctionInfo[] pythonFunctionInfos) {
    Class<?> clazz = CommonPythonUtil.loadClass(ARROW_PYTHON_OVER_WINDOW_AGGREGATE_FUNCTION_OPERATOR_NAME);
    RowType udfInputType = (RowType) Projection.of(udafInputOffsets).project(inputRowType);
    RowType udfOutputType = (RowType) Projection.range(inputRowType.getFieldCount(), outputRowType.getFieldCount()).project(outputRowType);
    PartitionSpec partitionSpec = overSpec.getPartition();
    List<OverSpec.GroupSpec> groups = overSpec.getGroups();
    SortSpec sortSpec = groups.get(groups.size() - 1).getSort();
    try {
        Constructor<?> ctor = clazz.getConstructor(Configuration.class, PythonFunctionInfo[].class, RowType.class, RowType.class, RowType.class, long[].class, long[].class, boolean[].class, int[].class, int.class, boolean.class, GeneratedProjection.class, GeneratedProjection.class, GeneratedProjection.class);
        return (OneInputStreamOperator<RowData, RowData>) ctor.newInstance(pythonConfig, pythonFunctionInfos, inputRowType, udfInputType, udfOutputType, lowerBoundary.stream().mapToLong(i -> i).toArray(), upperBoundary.stream().mapToLong(i -> i).toArray(), isRangeWindows, aggWindowIndex.stream().mapToInt(i -> i).toArray(), sortSpec.getFieldIndices()[0], sortSpec.getAscendingOrders()[0], ProjectionCodeGenerator.generateProjection(CodeGeneratorContext.apply(config.getTableConfig()), "UdafInputProjection", inputRowType, udfInputType, udafInputOffsets), ProjectionCodeGenerator.generateProjection(CodeGeneratorContext.apply(config.getTableConfig()), "GroupKey", inputRowType, (RowType) Projection.of(partitionSpec.getFieldIndices()).project(inputRowType), partitionSpec.getFieldIndices()), ProjectionCodeGenerator.generateProjection(CodeGeneratorContext.apply(config.getTableConfig()), "GroupSet", inputRowType, (RowType) Projection.of(partitionSpec.getFieldIndices()).project(inputRowType), partitionSpec.getFieldIndices()));
    } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new TableException("Python BatchArrowPythonOverWindowAggregateFunctionOperator constructed failed.", e);
    }
}

Example 5 with SortSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.SortSpec in project flink by apache.

the class SortCodeGeneratorTest method testOneKey.

@Test
public void testOneKey() throws Exception {
    for (int time = 0; time < 100; time++) {
        Random rnd = new Random();
        LogicalType[] fields = new LogicalType[rnd.nextInt(9) + 1];
        for (int i = 0; i < fields.length; i++) {
            fields[i] = types[rnd.nextInt(types.length)];
        }
        inputType = RowType.of(fields);
        SortSpec.SortSpecBuilder builder = SortSpec.builder();
        boolean order = rnd.nextBoolean();
        builder.addField(0, order, SortUtil.getNullDefaultOrder(order));
        sortSpec = builder.build();
        testInner();
    }
}