Examples with LogicalType org.apache.flink.table.types.logical.LogicalType used on opensource projects

Example 51 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class DefaultSchemaResolver method validateTimeColumn.

private Column validateTimeColumn(String columnName, List<Column> columns) {
    final Optional<Column> timeColumn = columns.stream().filter(c -> c.getName().equals(columnName)).findFirst();
    if (!timeColumn.isPresent()) {
        throw new ValidationException(String.format("Invalid column name '%s' for rowtime attribute in watermark declaration. Available columns are: %s", columnName, columns.stream().map(Column::getName).collect(Collectors.toList())));
    }
    final LogicalType timeFieldType = timeColumn.get().getDataType().getLogicalType();
    if (!canBeTimeAttributeType(timeFieldType) || getPrecision(timeFieldType) > 3) {
        throw new ValidationException(String.format("Invalid data type of time field for watermark definition. " + "The field must be of type TIMESTAMP(p) or TIMESTAMP_LTZ(p)," + " the supported precision 'p' is from 0 to 3, but the time field type is %s", timeFieldType));
    }
    if (isProctimeAttribute(timeFieldType)) {
        throw new ValidationException("A watermark can not be defined for a processing-time attribute.");
    }
    return timeColumn.get();
}

Example 52 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class SchemaTranslator method createConsumingResult.

/**
 * Converts the given {@link DataType} and an optional declared {@link Schema} (possibly
 * incomplete) into the final {@link ConsumingResult}.
 *
 * <p>This method serves three types of use cases:
 *
 * <ul>
 *   <li>1. Derive physical columns from the input data type.
 *   <li>2. Derive physical columns but merge them with declared computed columns and other
 *       schema information.
 *   <li>3. Derive and enrich physical columns and merge other schema information (only if
 *       {@param mergePhysicalSchema} is set to {@code true}).
 * </ul>
 */
public static ConsumingResult createConsumingResult(DataTypeFactory dataTypeFactory, DataType inputDataType, @Nullable Schema declaredSchema, boolean mergePhysicalSchema) {
    final LogicalType inputType = inputDataType.getLogicalType();
    // we don't allow modifying the number of columns during enrichment, therefore we preserve
    // whether the original type was qualified as a top-level record or not
    final boolean isTopLevelRecord = LogicalTypeChecks.isCompositeType(inputType);
    // the schema will be entirely derived from the input
    if (declaredSchema == null) {
        final Schema.Builder builder = Schema.newBuilder();
        addPhysicalSourceDataTypeFields(builder, inputDataType, null);
        return new ConsumingResult(inputDataType, isTopLevelRecord, builder.build(), null);
    }
    final List<UnresolvedColumn> declaredColumns = declaredSchema.getColumns();
    final UnresolvedPrimaryKey declaredPrimaryKey = declaredSchema.getPrimaryKey().orElse(null);
    // thus, it only enriches the non-physical column parts
    if (declaredColumns.stream().noneMatch(SchemaTranslator::isPhysical)) {
        final Schema.Builder builder = Schema.newBuilder();
        addPhysicalSourceDataTypeFields(builder, inputDataType, declaredPrimaryKey);
        builder.fromSchema(declaredSchema);
        return new ConsumingResult(inputDataType, isTopLevelRecord, builder.build(), null);
    }
    if (!mergePhysicalSchema) {
        return new ConsumingResult(inputDataType, isTopLevelRecord, declaredSchema, null);
    }
    // the declared schema enriches the physical data type and the derived schema,
    // it possibly projects the result
    final DataType patchedDataType = patchDataTypeFromDeclaredSchema(dataTypeFactory, inputDataType, declaredColumns);
    final Schema patchedSchema = createPatchedSchema(isTopLevelRecord, patchedDataType, declaredSchema);
    final List<String> projections = extractProjections(patchedSchema, declaredSchema);
    return new ConsumingResult(patchedDataType, isTopLevelRecord, patchedSchema, projections);
}

Example 53 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class PushPartitionIntoTableSourceScanRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    Filter filter = call.rel(0);
    LogicalTableScan scan = call.rel(1);
    TableSourceTable tableSourceTable = scan.getTable().unwrap(TableSourceTable.class);
    RelDataType inputFieldTypes = filter.getInput().getRowType();
    List<String> inputFieldNames = inputFieldTypes.getFieldNames();
    List<String> partitionFieldNames = tableSourceTable.contextResolvedTable().<ResolvedCatalogTable>getResolvedTable().getPartitionKeys();
    // extract partition predicates
    RelBuilder relBuilder = call.builder();
    RexBuilder rexBuilder = relBuilder.getRexBuilder();
    Tuple2<Seq<RexNode>, Seq<RexNode>> allPredicates = RexNodeExtractor.extractPartitionPredicateList(filter.getCondition(), FlinkRelOptUtil.getMaxCnfNodeCount(scan), inputFieldNames.toArray(new String[0]), rexBuilder, partitionFieldNames.toArray(new String[0]));
    RexNode partitionPredicate = RexUtil.composeConjunction(rexBuilder, JavaConversions.seqAsJavaList(allPredicates._1));
    if (partitionPredicate.isAlwaysTrue()) {
        return;
    }
    // build pruner
    LogicalType[] partitionFieldTypes = partitionFieldNames.stream().map(name -> {
        int index = inputFieldNames.indexOf(name);
        if (index < 0) {
            throw new TableException(String.format("Partitioned key '%s' isn't found in input columns. " + "Validator should have checked that.", name));
        }
        return inputFieldTypes.getFieldList().get(index).getType();
    }).map(FlinkTypeFactory::toLogicalType).toArray(LogicalType[]::new);
    RexNode finalPartitionPredicate = adjustPartitionPredicate(inputFieldNames, partitionFieldNames, partitionPredicate);
    FlinkContext context = ShortcutUtils.unwrapContext(scan);
    Function<List<Map<String, String>>, List<Map<String, String>>> defaultPruner = partitions -> PartitionPruner.prunePartitions(context.getTableConfig(), partitionFieldNames.toArray(new String[0]), partitionFieldTypes, partitions, finalPartitionPredicate);
    // prune partitions
    List<Map<String, String>> remainingPartitions = readPartitionsAndPrune(rexBuilder, context, tableSourceTable, defaultPruner, allPredicates._1(), inputFieldNames);
    // apply push down
    DynamicTableSource dynamicTableSource = tableSourceTable.tableSource().copy();
    PartitionPushDownSpec partitionPushDownSpec = new PartitionPushDownSpec(remainingPartitions);
    partitionPushDownSpec.apply(dynamicTableSource, SourceAbilityContext.from(scan));
    // build new statistic
    TableStats newTableStat = null;
    if (tableSourceTable.contextResolvedTable().isPermanent()) {
        ObjectIdentifier identifier = tableSourceTable.contextResolvedTable().getIdentifier();
        ObjectPath tablePath = identifier.toObjectPath();
        Catalog catalog = tableSourceTable.contextResolvedTable().getCatalog().get();
        for (Map<String, String> partition : remainingPartitions) {
            Optional<TableStats> partitionStats = getPartitionStats(catalog, tablePath, partition);
            if (!partitionStats.isPresent()) {
                // clear all information before
                newTableStat = null;
                break;
            } else {
                newTableStat = newTableStat == null ? partitionStats.get() : newTableStat.merge(partitionStats.get());
            }
        }
    }
    FlinkStatistic newStatistic = FlinkStatistic.builder().statistic(tableSourceTable.getStatistic()).tableStats(newTableStat).build();
    TableSourceTable newTableSourceTable = tableSourceTable.copy(dynamicTableSource, newStatistic, new SourceAbilitySpec[] { partitionPushDownSpec });
    LogicalTableScan newScan = LogicalTableScan.create(scan.getCluster(), newTableSourceTable, scan.getHints());
    // transform to new node
    RexNode nonPartitionPredicate = RexUtil.composeConjunction(rexBuilder, JavaConversions.seqAsJavaList(allPredicates._2()));
    if (nonPartitionPredicate.isAlwaysTrue()) {
        call.transformTo(newScan);
    } else {
        Filter newFilter = filter.copy(filter.getTraitSet(), newScan, nonPartitionPredicate);
        call.transformTo(newFilter);
    }
}

Example 54 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class WindowOperator method open.

@Override
public void open() throws Exception {
    super.open();
    collector = new TimestampedCollector<>(output);
    collector.eraseTimestamp();
    internalTimerService = getInternalTimerService("window-timers", windowSerializer, this);
    triggerContext = new TriggerContext();
    triggerContext.open();
    StateDescriptor<ValueState<RowData>, RowData> windowStateDescriptor = new ValueStateDescriptor<>("window-aggs", new RowDataSerializer(accumulatorTypes));
    this.windowState = (InternalValueState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
    if (produceUpdates) {
        LogicalType[] valueTypes = ArrayUtils.addAll(aggResultTypes, windowPropertyTypes);
        StateDescriptor<ValueState<RowData>, RowData> previousStateDescriptor = new ValueStateDescriptor<>("previous-aggs", new RowDataSerializer(valueTypes));
        this.previousState = (InternalValueState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, previousStateDescriptor);
    }
    compileGeneratedCode();
    WindowContext windowContext = new WindowContext();
    windowAggregator.open(new PerWindowStateDataViewStore(getKeyedStateBackend(), windowSerializer, getRuntimeContext()));
    if (windowAssigner instanceof MergingWindowAssigner) {
        this.windowFunction = new MergingWindowProcessFunction<>((MergingWindowAssigner<W>) windowAssigner, windowAggregator, windowSerializer, allowedLateness);
    } else if (windowAssigner instanceof PanedWindowAssigner) {
        this.windowFunction = new PanedWindowProcessFunction<>((PanedWindowAssigner<W>) windowAssigner, windowAggregator, allowedLateness);
    } else {
        this.windowFunction = new GeneralWindowProcessFunction<>(windowAssigner, windowAggregator, allowedLateness);
    }
    windowFunction.open(windowContext);
    // metrics
    this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
    this.lateRecordsDroppedRate = metrics.meter(LATE_ELEMENTS_DROPPED_RATE_METRIC_NAME, new MeterView(numLateRecordsDropped));
    this.watermarkLatency = metrics.gauge(WATERMARK_LATENCY_METRIC_NAME, () -> {
        long watermark = internalTimerService.currentWatermark();
        if (watermark < 0) {
            return 0L;
        } else {
            return internalTimerService.currentProcessingTime() - watermark;
        }
    });
}

Example 55 with LogicalType

use of org.apache.flink.table.types.logical.LogicalType in project flink by apache.

the class DecimalDivideTypeStrategy method inferType.

@Override
public Optional<DataType> inferType(CallContext callContext) {
    final List<DataType> argumentDataTypes = callContext.getArgumentDataTypes();
    final LogicalType dividend = argumentDataTypes.get(0).getLogicalType();
    final LogicalType divisor = argumentDataTypes.get(1).getLogicalType();
    // a hack to make legacy types possible until we drop them
    if (dividend instanceof LegacyTypeInformationType) {
        return Optional.of(argumentDataTypes.get(0));
    }
    if (divisor instanceof LegacyTypeInformationType) {
        return Optional.of(argumentDataTypes.get(1));
    }
    if (!isDecimalComputation(dividend, divisor)) {
        return Optional.empty();
    }
    final DecimalType decimalType = LogicalTypeMerging.findDivisionDecimalType(getPrecision(dividend), getScale(dividend), getPrecision(divisor), getScale(divisor));
    return Optional.of(fromLogicalToDataType(decimalType));
}