Examples with ResolvedSchema org.apache.flink.table.catalog.ResolvedSchema used on opensource projects

Example 61 with ResolvedSchema

use of org.apache.flink.table.catalog.ResolvedSchema in project flink by apache.

the class KinesisDynamicTableSinkFactoryTest method testBadTableSinkForNonExistingPartitionerClass.

@Test
public void testBadTableSinkForNonExistingPartitionerClass() {
    ResolvedSchema sinkSchema = defaultSinkSchema();
    Map<String, String> sinkOptions = defaultTableOptions().withTableOption(KinesisConnectorOptions.SINK_PARTITIONER, "abc").build();
    Assertions.assertThatExceptionOfType(ValidationException.class).isThrownBy(() -> createTableSink(sinkSchema, sinkOptions)).havingCause().withMessageContaining("Could not find and instantiate partitioner class 'abc'");
}

Example 62 with ResolvedSchema

use of org.apache.flink.table.catalog.ResolvedSchema in project flink by apache.

the class KinesisDynamicTableSinkFactoryTest method testGoodTableSinkForNonPartitionedTableWithProducerOptions.

@Test
public void testGoodTableSinkForNonPartitionedTableWithProducerOptions() {
    ResolvedSchema sinkSchema = defaultSinkSchema();
    Map<String, String> sinkOptions = defaultTableOptionsWithDeprecatedOptions().build();
    // Construct actual DynamicTableSink using FactoryUtil
    KinesisDynamicSink actualSink = (KinesisDynamicSink) createTableSink(sinkSchema, sinkOptions);
    // Construct expected DynamicTableSink using factory under test
    KinesisDynamicSink expectedSink = (KinesisDynamicSink) new KinesisDynamicSink.KinesisDynamicTableSinkBuilder().setFailOnError(true).setMaxBatchSize(100).setMaxInFlightRequests(100).setMaxTimeInBufferMS(1000).setConsumedDataType(sinkSchema.toPhysicalRowDataType()).setStream(STREAM_NAME).setKinesisClientProperties(defaultProducerProperties()).setEncodingFormat(new TestFormatFactory.EncodingFormatMock(",")).setPartitioner(new RandomKinesisPartitionKeyGenerator<>()).build();
    // verify that the constructed DynamicTableSink is as expected
    Assertions.assertThat(actualSink).isEqualTo(expectedSink);
    // verify the produced sink
    DynamicTableSink.SinkRuntimeProvider sinkFunctionProvider = actualSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
    Sink<RowData> sinkFunction = ((SinkV2Provider) sinkFunctionProvider).createSink();
    Assertions.assertThat(sinkFunction).isInstanceOf(KinesisDataStreamsSink.class);
}

Example 63 with ResolvedSchema

use of org.apache.flink.table.catalog.ResolvedSchema in project flink by apache.

the class ElasticsearchDynamicSinkBaseITCase method testWritingDocuments.

@Test
public void testWritingDocuments() throws Exception {
    ResolvedSchema schema = new ResolvedSchema(Arrays.asList(Column.physical("a", DataTypes.BIGINT().notNull()), Column.physical("b", DataTypes.TIME()), Column.physical("c", DataTypes.STRING().notNull()), Column.physical("d", DataTypes.FLOAT()), Column.physical("e", DataTypes.TINYINT().notNull()), Column.physical("f", DataTypes.DATE()), Column.physical("g", DataTypes.TIMESTAMP().notNull())), Collections.emptyList(), UniqueConstraint.primaryKey("name", Arrays.asList("a", "g")));
    GenericRowData rowData = GenericRowData.of(1L, 12345, StringData.fromString("ABCDE"), 12.12f, (byte) 2, 12345, TimestampData.fromLocalDateTime(LocalDateTime.parse("2012-12-12T12:12:12")));
    String index = "writing-documents";
    ElasticsearchDynamicSinkFactoryBase sinkFactory = getDynamicSinkFactory();
    DynamicTableSink.SinkRuntimeProvider runtimeProvider = sinkFactory.createDynamicTableSink(getPrefilledTestContext(index).withSchema(schema).build()).getSinkRuntimeProvider(new ElasticsearchUtil.MockContext());
    final SinkV2Provider sinkProvider = (SinkV2Provider) runtimeProvider;
    final Sink<RowData> sink = sinkProvider.createSink();
    StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
    environment.setParallelism(4);
    rowData.setRowKind(RowKind.UPDATE_AFTER);
    environment.<RowData>fromElements(rowData).sinkTo(sink);
    environment.execute();
    RestHighLevelClient client = getClient();
    Map<String, Object> response = makeGetRequest(client, index, "1_2012-12-12T12:12:12");
    Map<Object, Object> expectedMap = new HashMap<>();
    expectedMap.put("a", 1);
    expectedMap.put("b", "00:00:12");
    expectedMap.put("c", "ABCDE");
    expectedMap.put("d", 12.12d);
    expectedMap.put("e", 2);
    expectedMap.put("f", "2003-10-20");
    expectedMap.put("g", "2012-12-12 12:12:12");
    Assertions.assertEquals(response, expectedMap);
}

Example 64 with ResolvedSchema

use of org.apache.flink.table.catalog.ResolvedSchema in project flink by apache.

the class HBaseDynamicTableFactoryTest method testUnknownOption.

@Test
public void testUnknownOption() {
    Map<String, String> options = getAllOptions();
    options.put("sink.unknown.key", "unknown-value");
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(ROWKEY, STRING()), Column.physical(FAMILY1, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, INT()))));
    try {
        createTableSource(schema, options);
        fail("Should fail");
    } catch (Exception e) {
        assertTrue(ExceptionUtils.findThrowableWithMessage(e, "Unsupported options:\n\nsink.unknown.key").isPresent());
    }
    try {
        createTableSink(schema, options);
        fail("Should fail");
    } catch (Exception e) {
        assertTrue(ExceptionUtils.findThrowableWithMessage(e, "Unsupported options:\n\nsink.unknown.key").isPresent());
    }
}

Example 65 with ResolvedSchema

use of org.apache.flink.table.catalog.ResolvedSchema in project flink by apache.

the class HBaseDynamicTableFactoryTest method testTableSourceFactory.

@SuppressWarnings("rawtypes")
@Test
public void testTableSourceFactory() {
    ResolvedSchema schema = ResolvedSchema.of(Column.physical(FAMILY1, ROW(FIELD(COL1, INT()))), Column.physical(FAMILY2, ROW(FIELD(COL1, INT()), FIELD(COL2, BIGINT()))), Column.physical(ROWKEY, BIGINT()), Column.physical(FAMILY3, ROW(FIELD(COL1, DOUBLE()), FIELD(COL2, BOOLEAN()), FIELD(COL3, STRING()))), Column.physical(FAMILY4, ROW(FIELD(COL1, DECIMAL(10, 3)), FIELD(COL2, TIMESTAMP(3)), FIELD(COL3, DATE()), FIELD(COL4, TIME()))));
    DynamicTableSource source = createTableSource(schema, getAllOptions());
    assertTrue(source instanceof HBaseDynamicTableSource);
    HBaseDynamicTableSource hbaseSource = (HBaseDynamicTableSource) source;
    int[][] lookupKey = { { 2 } };
    LookupTableSource.LookupRuntimeProvider lookupProvider = hbaseSource.getLookupRuntimeProvider(new LookupRuntimeProviderContext(lookupKey));
    assertTrue(lookupProvider instanceof TableFunctionProvider);
    TableFunction tableFunction = ((TableFunctionProvider) lookupProvider).createTableFunction();
    assertTrue(tableFunction instanceof HBaseRowDataLookupFunction);
    assertEquals("testHBastTable", ((HBaseRowDataLookupFunction) tableFunction).getHTableName());
    HBaseTableSchema hbaseSchema = hbaseSource.getHBaseTableSchema();
    assertEquals(2, hbaseSchema.getRowKeyIndex());
    assertEquals(Optional.of(Types.LONG), hbaseSchema.getRowKeyTypeInfo());
    assertArrayEquals(new String[] { "f1", "f2", "f3", "f4" }, hbaseSchema.getFamilyNames());
    assertArrayEquals(new String[] { "c1" }, hbaseSchema.getQualifierNames("f1"));
    assertArrayEquals(new String[] { "c1", "c2" }, hbaseSchema.getQualifierNames("f2"));
    assertArrayEquals(new String[] { "c1", "c2", "c3" }, hbaseSchema.getQualifierNames("f3"));
    assertArrayEquals(new String[] { "c1", "c2", "c3", "c4" }, hbaseSchema.getQualifierNames("f4"));
    assertArrayEquals(new DataType[] { INT() }, hbaseSchema.getQualifierDataTypes("f1"));
    assertArrayEquals(new DataType[] { INT(), BIGINT() }, hbaseSchema.getQualifierDataTypes("f2"));
    assertArrayEquals(new DataType[] { DOUBLE(), BOOLEAN(), STRING() }, hbaseSchema.getQualifierDataTypes("f3"));
    assertArrayEquals(new DataType[] { DECIMAL(10, 3), TIMESTAMP(3), DATE(), TIME() }, hbaseSchema.getQualifierDataTypes("f4"));
}