Examples with LogicalType org.apache.flink.table.types.logical.LogicalType used on opensource projects

Example 46 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class BatchExecHashJoin method translateToPlanInternal.

@Override
@SuppressWarnings("unchecked")
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    ExecEdge leftInputEdge = getInputEdges().get(0);
    ExecEdge rightInputEdge = getInputEdges().get(1);
    Transformation<RowData> leftInputTransform = (Transformation<RowData>) leftInputEdge.translateToPlan(planner);
    Transformation<RowData> rightInputTransform = (Transformation<RowData>) rightInputEdge.translateToPlan(planner);
    // get input types
    RowType leftType = (RowType) leftInputEdge.getOutputType();
    RowType rightType = (RowType) rightInputEdge.getOutputType();
    JoinUtil.validateJoinSpec(joinSpec, leftType, rightType, false);
    int[] leftKeys = joinSpec.getLeftKeys();
    int[] rightKeys = joinSpec.getRightKeys();
    LogicalType[] keyFieldTypes = IntStream.of(leftKeys).mapToObj(leftType::getTypeAt).toArray(LogicalType[]::new);
    RowType keyType = RowType.of(keyFieldTypes);
    GeneratedJoinCondition condFunc = JoinUtil.generateConditionFunction(config.getTableConfig(), joinSpec.getNonEquiCondition().orElse(null), leftType, rightType);
    // projection for equals
    GeneratedProjection leftProj = ProjectionCodeGenerator.generateProjection(new CodeGeneratorContext(config.getTableConfig()), "HashJoinLeftProjection", leftType, keyType, leftKeys);
    GeneratedProjection rightProj = ProjectionCodeGenerator.generateProjection(new CodeGeneratorContext(config.getTableConfig()), "HashJoinRightProjection", rightType, keyType, rightKeys);
    Transformation<RowData> buildTransform;
    Transformation<RowData> probeTransform;
    GeneratedProjection buildProj;
    GeneratedProjection probeProj;
    int[] buildKeys;
    int[] probeKeys;
    RowType buildType;
    RowType probeType;
    int buildRowSize;
    long buildRowCount;
    long probeRowCount;
    boolean reverseJoin = !leftIsBuild;
    if (leftIsBuild) {
        buildTransform = leftInputTransform;
        buildProj = leftProj;
        buildType = leftType;
        buildRowSize = estimatedLeftAvgRowSize;
        buildRowCount = estimatedLeftRowCount;
        buildKeys = leftKeys;
        probeTransform = rightInputTransform;
        probeProj = rightProj;
        probeType = rightType;
        probeRowCount = estimatedLeftRowCount;
        probeKeys = rightKeys;
    } else {
        buildTransform = rightInputTransform;
        buildProj = rightProj;
        buildType = rightType;
        buildRowSize = estimatedRightAvgRowSize;
        buildRowCount = estimatedRightRowCount;
        buildKeys = rightKeys;
        probeTransform = leftInputTransform;
        probeProj = leftProj;
        probeType = leftType;
        probeRowCount = estimatedLeftRowCount;
        probeKeys = leftKeys;
    }
    // operator
    StreamOperatorFactory<RowData> operator;
    FlinkJoinType joinType = joinSpec.getJoinType();
    HashJoinType hashJoinType = HashJoinType.of(leftIsBuild, joinType.isLeftOuter(), joinType.isRightOuter(), joinType == FlinkJoinType.SEMI, joinType == FlinkJoinType.ANTI);
    if (LongHashJoinGenerator.support(hashJoinType, keyType, joinSpec.getFilterNulls())) {
        operator = LongHashJoinGenerator.gen(config.getTableConfig(), hashJoinType, keyType, buildType, probeType, buildKeys, probeKeys, buildRowSize, buildRowCount, reverseJoin, condFunc);
    } else {
        operator = SimpleOperatorFactory.of(HashJoinOperator.newHashJoinOperator(hashJoinType, condFunc, reverseJoin, joinSpec.getFilterNulls(), buildProj, probeProj, tryDistinctBuildRow, buildRowSize, buildRowCount, probeRowCount, keyType));
    }
    long managedMemory = config.get(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_HASH_JOIN_MEMORY).getBytes();
    return ExecNodeUtil.createTwoInputTransformation(buildTransform, probeTransform, createTransformationName(config), createTransformationDescription(config), operator, InternalTypeInfo.of(getOutputType()), probeTransform.getParallelism(), managedMemory);
}

Example 47 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class WindowOperatorContractTest method createWindowOperator.

// ------------------------------------------------------------------------------------------
@SuppressWarnings("unchecked")
private <W extends Window> KeyedOneInputStreamOperatorTestHarness<RowData, RowData, RowData> createWindowOperator(WindowAssigner<W> assigner, Trigger<W> trigger, NamespaceAggsHandleFunctionBase<W> aggregationsFunction, long allowedLateness) throws Exception {
    LogicalType[] inputTypes = new LogicalType[] { VarCharType.STRING_TYPE, new IntType() };
    RowDataKeySelector keySelector = HandwrittenSelectorUtil.getRowDataSelector(new int[] { 0 }, inputTypes);
    TypeInformation<RowData> keyType = keySelector.getProducedType();
    LogicalType[] accTypes = new LogicalType[] { new BigIntType(), new BigIntType() };
    LogicalType[] windowTypes = new LogicalType[] { new BigIntType(), new BigIntType() };
    LogicalType[] outputTypeWithoutKeys = new LogicalType[] { new BigIntType(), new BigIntType(), new BigIntType(), new BigIntType() };
    boolean sendRetraction = allowedLateness > 0;
    if (aggregationsFunction instanceof NamespaceAggsHandleFunction) {
        AggregateWindowOperator operator = new AggregateWindowOperator((NamespaceAggsHandleFunction) aggregationsFunction, mock(RecordEqualiser.class), assigner, trigger, assigner.getWindowSerializer(new ExecutionConfig()), inputTypes, outputTypeWithoutKeys, accTypes, windowTypes, 2, sendRetraction, allowedLateness, UTC_ZONE_ID, -1);
        return new KeyedOneInputStreamOperatorTestHarness<RowData, RowData, RowData>(operator, keySelector, keyType);
    } else {
        TableAggregateWindowOperator operator = new TableAggregateWindowOperator((NamespaceTableAggsHandleFunction) aggregationsFunction, assigner, trigger, assigner.getWindowSerializer(new ExecutionConfig()), inputTypes, outputTypeWithoutKeys, accTypes, windowTypes, 2, sendRetraction, allowedLateness, UTC_ZONE_ID, -1);
        return new KeyedOneInputStreamOperatorTestHarness<RowData, RowData, RowData>(operator, keySelector, keyType);
    }
}

Example 48 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class WindowOperatorTest method testTumblingCountWindow.

@Test
public void testTumblingCountWindow() throws Exception {
    if (!UTC_ZONE_ID.equals(shiftTimeZone)) {
        return;
    }
    closeCalled.set(0);
    final int windowSize = 3;
    LogicalType[] windowTypes = new LogicalType[] { new BigIntType() };
    WindowOperator operator = WindowOperatorBuilder.builder().withInputFields(inputFieldTypes).withShiftTimezone(shiftTimeZone).countWindow(windowSize).aggregateAndBuild(getCountWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(insertRecord("key2", 1, 0L));
    testHarness.processElement(insertRecord("key2", 2, 1000L));
    testHarness.processElement(insertRecord("key2", 3, 2500L));
    testHarness.processElement(insertRecord("key1", 1, 10L));
    testHarness.processElement(insertRecord("key1", 2, 1000L));
    testHarness.processWatermark(new Watermark(12000));
    testHarness.setProcessingTime(12000L);
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 6L, 3L, 0L)));
    expectedOutput.add(new Watermark(12000));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    // do a snapshot, close and restore again
    OperatorSubtaskState snapshotV2 = testHarness.snapshot(0L, 0);
    testHarness.close();
    expectedOutput.clear();
    testHarness = createTestHarness(operator);
    testHarness.setup();
    testHarness.initializeState(snapshotV2);
    testHarness.open();
    testHarness.processElement(insertRecord("key1", 2, 2500L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 5L, 3L, 0L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(insertRecord("key2", 4, 5501L));
    testHarness.processElement(insertRecord("key2", 5, 6000L));
    testHarness.processElement(insertRecord("key2", 5, 6000L));
    testHarness.processElement(insertRecord("key2", 6, 6050L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 14L, 3L, 1L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(insertRecord("key1", 3, 4000L));
    testHarness.processElement(insertRecord("key2", 10, 15000L));
    testHarness.processElement(insertRecord("key2", 20, 15000L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 36L, 3L, 2L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(insertRecord("key1", 2, 2500L));
    testHarness.processElement(insertRecord("key1", 2, 2500L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 7L, 3L, 1L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
    // we close once in the rest...
    assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 49 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class DefaultSchemaResolver method adjustRowtimeAttribute.

private Column adjustRowtimeAttribute(List<WatermarkSpec> watermarkSpecs, Column column) {
    final String name = column.getName();
    final DataType dataType = column.getDataType();
    final boolean hasWatermarkSpec = watermarkSpecs.stream().anyMatch(s -> s.getRowtimeAttribute().equals(name));
    if (hasWatermarkSpec && isStreamingMode) {
        switch(dataType.getLogicalType().getTypeRoot()) {
            case TIMESTAMP_WITHOUT_TIME_ZONE:
                final TimestampType originalType = (TimestampType) dataType.getLogicalType();
                final LogicalType rowtimeType = new TimestampType(originalType.isNullable(), TimestampKind.ROWTIME, originalType.getPrecision());
                return column.copy(replaceLogicalType(dataType, rowtimeType));
            case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
                final LocalZonedTimestampType timestampLtzType = (LocalZonedTimestampType) dataType.getLogicalType();
                final LogicalType rowtimeLtzType = new LocalZonedTimestampType(timestampLtzType.isNullable(), TimestampKind.ROWTIME, timestampLtzType.getPrecision());
                return column.copy(replaceLogicalType(dataType, rowtimeLtzType));
            default:
                throw new ValidationException("Invalid data type of expression for rowtime definition. " + "The field must be of type TIMESTAMP(p) or TIMESTAMP_LTZ(p)," + " the supported precision 'p' is from 0 to 3.");
        }
    }
    return column;
}

Example 50 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class DefaultSchemaResolver method resolveWatermarkSpecs.

private List<WatermarkSpec> resolveWatermarkSpecs(List<UnresolvedWatermarkSpec> unresolvedWatermarkSpecs, List<Column> inputColumns) {
    if (unresolvedWatermarkSpecs.size() == 0) {
        return Collections.emptyList();
    }
    if (unresolvedWatermarkSpecs.size() > 1) {
        throw new ValidationException("Multiple watermark definitions are not supported yet.");
    }
    final UnresolvedWatermarkSpec watermarkSpec = unresolvedWatermarkSpecs.get(0);
    // validate time attribute
    final String timeColumn = watermarkSpec.getColumnName();
    final Column validatedTimeColumn = validateTimeColumn(timeColumn, inputColumns);
    // resolve watermark expression
    final ResolvedExpression watermarkExpression;
    try {
        watermarkExpression = resolveExpression(inputColumns, watermarkSpec.getWatermarkExpression(), validatedTimeColumn.getDataType());
    } catch (Exception e) {
        throw new ValidationException(String.format("Invalid expression for watermark '%s'.", watermarkSpec.toString()), e);
    }
    final LogicalType outputType = watermarkExpression.getOutputDataType().getLogicalType();
    final LogicalType timeColumnType = validatedTimeColumn.getDataType().getLogicalType();
    validateWatermarkExpression(outputType);
    if (outputType.getTypeRoot() != timeColumnType.getTypeRoot()) {
        throw new ValidationException(String.format("The watermark declaration's output data type '%s' is different " + "from the time field's data type '%s'.", outputType, timeColumnType));
    }
    return Collections.singletonList(WatermarkSpec.of(watermarkSpec.getColumnName(), watermarkExpression));
}