Examples with WatermarkSpec org.apache.flink.table.api.WatermarkSpec used on opensource projects

Example 1 with WatermarkSpec

use of org.apache.flink.table.api.WatermarkSpec in project flink by apache.

the class DescriptorProperties method putTableSchema.

/**
 * Adds a table schema under the given key.
 */
public void putTableSchema(String key, TableSchema schema) {
    checkNotNull(key);
    checkNotNull(schema);
    final String[] fieldNames = schema.getFieldNames();
    final DataType[] fieldTypes = schema.getFieldDataTypes();
    final String[] fieldExpressions = schema.getTableColumns().stream().map(column -> {
        if (column instanceof ComputedColumn) {
            return ((ComputedColumn) column).getExpression();
        }
        return null;
    }).toArray(String[]::new);
    final String[] fieldMetadata = schema.getTableColumns().stream().map(column -> {
        if (column instanceof MetadataColumn) {
            return ((MetadataColumn) column).getMetadataAlias().orElse(column.getName());
        }
        return null;
    }).toArray(String[]::new);
    final String[] fieldVirtual = schema.getTableColumns().stream().map(column -> {
        if (column instanceof MetadataColumn) {
            return Boolean.toString(((MetadataColumn) column).isVirtual());
        }
        return null;
    }).toArray(String[]::new);
    final List<List<String>> values = new ArrayList<>();
    for (int i = 0; i < schema.getFieldCount(); i++) {
        values.add(Arrays.asList(fieldNames[i], fieldTypes[i].getLogicalType().asSerializableString(), fieldExpressions[i], fieldMetadata[i], fieldVirtual[i]));
    }
    putIndexedOptionalProperties(key, Arrays.asList(NAME, DATA_TYPE, EXPR, METADATA, VIRTUAL), values);
    if (!schema.getWatermarkSpecs().isEmpty()) {
        final List<List<String>> watermarkValues = new ArrayList<>();
        for (WatermarkSpec spec : schema.getWatermarkSpecs()) {
            watermarkValues.add(Arrays.asList(spec.getRowtimeAttribute(), spec.getWatermarkExpr(), spec.getWatermarkExprOutputType().getLogicalType().asSerializableString()));
        }
        putIndexedFixedProperties(key + '.' + WATERMARK, Arrays.asList(WATERMARK_ROWTIME, WATERMARK_STRATEGY_EXPR, WATERMARK_STRATEGY_DATA_TYPE), watermarkValues);
    }
    schema.getPrimaryKey().ifPresent(pk -> {
        putString(key + '.' + PRIMARY_KEY_NAME, pk.getName());
        putString(key + '.' + PRIMARY_KEY_COLUMNS, String.join(",", pk.getColumns()));
    });
}

Example 2 with WatermarkSpec

use of org.apache.flink.table.api.WatermarkSpec in project flink by apache.

the class TestValuesTableFactory method validateAndExtractRowtimeIndex.

private static int validateAndExtractRowtimeIndex(CatalogTable sinkTable, boolean dropLateEvent, boolean isInsertOnly) {
    if (!dropLateEvent) {
        return -1;
    } else if (!isInsertOnly) {
        throw new ValidationException("Option 'sink.drop-late-event' only works for insert-only sink now.");
    }
    TableSchema schema = sinkTable.getSchema();
    List<WatermarkSpec> watermarkSpecs = schema.getWatermarkSpecs();
    if (watermarkSpecs.size() == 0) {
        throw new ValidationException("Please define the watermark in the schema that is used to indicate the rowtime column. " + "The sink function will compare the rowtime and the current watermark to determine whether the event is late.");
    }
    String rowtimeName = watermarkSpecs.get(0).getRowtimeAttribute();
    return Arrays.asList(schema.getFieldNames()).indexOf(rowtimeName);
}

Example 3 with WatermarkSpec

use of org.apache.flink.table.api.WatermarkSpec in project flink by apache.

the class TableSchemaUtils method builderWithGivenSchema.

/**
 * Creates a builder with given table schema.
 *
 * @param oriSchema Original schema
 * @return the builder with all the information from the given schema
 */
public static TableSchema.Builder builderWithGivenSchema(TableSchema oriSchema) {
    TableSchema.Builder builder = builderWithGivenColumns(oriSchema.getTableColumns());
    // Copy watermark specification.
    for (WatermarkSpec wms : oriSchema.getWatermarkSpecs()) {
        builder.watermark(wms.getRowtimeAttribute(), wms.getWatermarkExpr(), wms.getWatermarkExprOutputType());
    }
    // Copy primary key constraint.
    oriSchema.getPrimaryKey().map(pk -> builder.primaryKey(pk.getName(), pk.getColumns().toArray(new String[0])));
    return builder;
}

Example 4 with WatermarkSpec

use of org.apache.flink.table.api.WatermarkSpec in project flink by apache.

the class TableSchemaUtils method dropConstraint.

/**
 * Creates a new schema but drop the constraint with given name.
 */
public static TableSchema dropConstraint(TableSchema oriSchema, String constraintName) {
    // Validate the constraint name is valid.
    Optional<UniqueConstraint> uniqueConstraintOpt = oriSchema.getPrimaryKey();
    if (!uniqueConstraintOpt.isPresent() || !uniqueConstraintOpt.get().getName().equals(constraintName)) {
        throw new ValidationException(String.format("Constraint %s to drop does not exist", constraintName));
    }
    TableSchema.Builder builder = builderWithGivenColumns(oriSchema.getTableColumns());
    // Copy watermark specification.
    for (WatermarkSpec wms : oriSchema.getWatermarkSpecs()) {
        builder.watermark(wms.getRowtimeAttribute(), wms.getWatermarkExpr(), wms.getWatermarkExprOutputType());
    }
    return builder.build();
}