Examples with DynamicTableSource org.apache.flink.table.connector.source.DynamicTableSource used on opensource projects

Example 41 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class HiveDynamicTableFactoryTest method testHiveLookupSourceOptions.

@Test
public void testHiveLookupSourceOptions() throws Exception {
    // test hive bounded source is a lookup source
    tableEnv.executeSql(String.format("create table table5 (x int, y string, z int) tblproperties ('%s'='5min')", LOOKUP_JOIN_CACHE_TTL.key()));
    DynamicTableSource tableSource1 = getTableSource("table5");
    assertTrue(tableSource1 instanceof HiveLookupTableSource);
    // test hive streaming source is a lookup source when 'streaming-source.partition.include' =
    // 'latest'
    tableEnv.executeSql(String.format("create table table6 (x int, y string, z int)" + " tblproperties ('%s' = 'true', '%s' = 'latest')", STREAMING_SOURCE_ENABLE.key(), STREAMING_SOURCE_PARTITION_INCLUDE.key()));
    DynamicTableSource tableSource2 = getTableSource("table6");
    assertTrue(tableSource2 instanceof HiveLookupTableSource);
    FileSystemLookupFunction lookupFunction = (FileSystemLookupFunction) ((HiveLookupTableSource) tableSource2).getLookupFunction(new int[][] { { 0 } });
    // test default lookup cache ttl for streaming-source is 1 hour
    assertEquals(Duration.ofHours(1), lookupFunction.getReloadInterval());
    HiveLookupTableSource lookupTableSource = (HiveLookupTableSource) tableSource2;
    Configuration configuration = new Configuration();
    lookupTableSource.catalogTable.getOptions().forEach(configuration::setString);
    assertEquals(configuration.get(STREAMING_SOURCE_PARTITION_ORDER), HiveOptions.PartitionOrder.PARTITION_NAME);
    // test lookup with partition-time extractor options
    tableEnv.executeSql(String.format("create table table7 (x int, y string, z int)" + " tblproperties (" + "'%s' = 'true'," + " '%s' = 'latest'," + " '%s' = '120min'," + " '%s' = 'partition-time', " + " '%s' = 'custom'," + " '%s' = 'path.to..TimeExtractor')", STREAMING_SOURCE_ENABLE.key(), STREAMING_SOURCE_PARTITION_INCLUDE.key(), STREAMING_SOURCE_MONITOR_INTERVAL.key(), STREAMING_SOURCE_PARTITION_ORDER.key(), PARTITION_TIME_EXTRACTOR_KIND.key(), PARTITION_TIME_EXTRACTOR_CLASS.key()));
    DynamicTableSource tableSource3 = getTableSource("table7");
    assertTrue(tableSource3 instanceof HiveLookupTableSource);
    HiveLookupTableSource tableSource = (HiveLookupTableSource) tableSource3;
    Configuration configuration1 = new Configuration();
    tableSource.catalogTable.getOptions().forEach(configuration1::setString);
    assertEquals(configuration1.get(STREAMING_SOURCE_PARTITION_ORDER), HiveOptions.PartitionOrder.PARTITION_TIME);
    assertEquals(configuration1.get(PARTITION_TIME_EXTRACTOR_KIND), "custom");
    assertEquals(configuration1.get(PARTITION_TIME_EXTRACTOR_CLASS), "path.to..TimeExtractor");
    tableEnv.executeSql(String.format("create table table8 (x int, y string, z int)" + " tblproperties ('%s' = 'true', '%s' = 'latest', '%s' = '5min')", STREAMING_SOURCE_ENABLE.key(), STREAMING_SOURCE_PARTITION_INCLUDE.key(), STREAMING_SOURCE_MONITOR_INTERVAL.key()));
    DynamicTableSource tableSource4 = getTableSource("table8");
    assertTrue(tableSource4 instanceof HiveLookupTableSource);
    HiveLookupTableSource lookupTableSource4 = (HiveLookupTableSource) tableSource4;
    Configuration configuration4 = new Configuration();
    lookupTableSource4.catalogTable.getOptions().forEach(configuration4::setString);
    assertEquals(configuration4.get(STREAMING_SOURCE_MONITOR_INTERVAL), Duration.ofMinutes(5L));
}

Example 42 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class HiveDynamicTableFactoryTest method testHiveStreamingSourceOptions.

@Test
public void testHiveStreamingSourceOptions() throws Exception {
    // test default hive streaming-source is not a lookup source
    tableEnv.executeSql(String.format("create table table1 (x int, y string, z int) partitioned by (" + " pt_year int, pt_mon string, pt_day string)" + " tblproperties ('%s' = 'true')", STREAMING_SOURCE_ENABLE.key()));
    DynamicTableSource tableSource1 = getTableSource("table1");
    assertFalse(tableSource1 instanceof HiveLookupTableSource);
    HiveTableSource tableSource = (HiveTableSource) tableSource1;
    Configuration configuration = new Configuration();
    tableSource.catalogTable.getOptions().forEach(configuration::setString);
    assertEquals(HiveOptions.PartitionOrder.PARTITION_NAME, configuration.get(STREAMING_SOURCE_PARTITION_ORDER));
    // test table can't be selected when set 'streaming-source.partition.include' to 'latest'
    tableEnv.executeSql(String.format("create table table2 (x int, y string, z int) partitioned by (" + " pt_year int, pt_mon string, pt_day string)" + " tblproperties ('%s' = 'true', '%s' = 'latest')", STREAMING_SOURCE_ENABLE.key(), STREAMING_SOURCE_PARTITION_INCLUDE.key()));
    DynamicTableSource tableSource2 = getTableSource("table2");
    assertTrue(tableSource2 instanceof HiveLookupTableSource);
    try {
        tableEnv.executeSql("select * from table2");
    } catch (Throwable t) {
        assertTrue(ExceptionUtils.findThrowableWithMessage(t, "The only supported 'streaming-source.partition.include' is 'all' in" + " hive table scan, but is 'latest'").isPresent());
    }
    // test table support 'partition-name' in option 'streaming-source.partition.order'.
    tableEnv.executeSql(String.format("create table table3 (x int, y string, z int) partitioned by (" + " pt_year int, pt_mon string, pt_day string)" + " tblproperties ('%s' = 'true', '%s' = 'partition-name')", STREAMING_SOURCE_ENABLE.key(), STREAMING_SOURCE_PARTITION_ORDER.key()));
    DynamicTableSource tableSource3 = getTableSource("table3");
    assertTrue(tableSource3 instanceof HiveTableSource);
    HiveTableSource hiveTableSource3 = (HiveTableSource) tableSource3;
    Configuration configuration1 = new Configuration();
    hiveTableSource3.catalogTable.getOptions().forEach(configuration1::setString);
    PartitionOrder partitionOrder1 = configuration1.get(STREAMING_SOURCE_PARTITION_ORDER);
    assertEquals(HiveOptions.PartitionOrder.PARTITION_NAME, partitionOrder1);
    // test deprecated option key 'streaming-source.consume-order' and new key
    // 'streaming-source.partition-order'
    tableEnv.executeSql(String.format("create table table4 (x int, y string, z int) partitioned by (" + " pt_year int, pt_mon string, pt_day string)" + " tblproperties ('%s' = 'true', '%s' = 'partition-time')", STREAMING_SOURCE_ENABLE.key(), "streaming-source.consume-order"));
    DynamicTableSource tableSource4 = getTableSource("table4");
    assertTrue(tableSource4 instanceof HiveTableSource);
    HiveTableSource hiveTableSource = (HiveTableSource) tableSource4;
    Configuration configuration2 = new Configuration();
    hiveTableSource.catalogTable.getOptions().forEach(configuration2::setString);
    PartitionOrder partitionOrder2 = configuration2.get(STREAMING_SOURCE_PARTITION_ORDER);
    assertEquals(HiveOptions.PartitionOrder.PARTITION_TIME, partitionOrder2);
}

Example 43 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class HiveDynamicTableFactoryTest method testJobConfWithCredentials.

@Test
public void testJobConfWithCredentials() throws Exception {
    final Text hdfsDelegationTokenKind = new Text("HDFS_DELEGATION_TOKEN");
    final Text hdfsDelegationTokenService = new Text("ha-hdfs:hadoop-namespace");
    Credentials credentials = new Credentials();
    credentials.addToken(hdfsDelegationTokenService, new Token<>(new byte[4], new byte[4], hdfsDelegationTokenKind, hdfsDelegationTokenService));
    UserGroupInformation.getCurrentUser().addCredentials(credentials);
    // test table source's jobConf with credentials
    tableEnv.executeSql(String.format("create table table10 (x int, y string, z int) partitioned by (" + " pt_year int, pt_mon string, pt_day string)"));
    DynamicTableSource tableSource1 = getTableSource("table10");
    HiveTableSource tableSource = (HiveTableSource) tableSource1;
    Token token = tableSource.getJobConf().getCredentials().getToken(hdfsDelegationTokenService);
    assertNotNull(token);
    assertEquals(hdfsDelegationTokenKind, token.getKind());
    assertEquals(hdfsDelegationTokenService, token.getService());
    // test table sink's jobConf with credentials
    DynamicTableSink tableSink1 = getTableSink("table10");
    HiveTableSink tableSink = (HiveTableSink) tableSink1;
    token = tableSink.getJobConf().getCredentials().getToken(hdfsDelegationTokenService);
    assertNotNull(token);
    assertEquals(hdfsDelegationTokenKind, token.getKind());
    assertEquals(hdfsDelegationTokenService, token.getService());
}

Example 44 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class CsvFormatFactoryTest method testDeserializeWithEscapedFieldDelimiter.

@Test
public void testDeserializeWithEscapedFieldDelimiter() throws IOException {
    // test deserialization schema
    final Map<String, String> options = getModifiedOptions(opts -> opts.put("csv.field-delimiter", "\t"));
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
    TestDynamicTableFactory.DynamicTableSourceMock sourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> deserializationSchema = sourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, PHYSICAL_DATA_TYPE);
    RowData expected = GenericRowData.of(fromString("abc"), 123, false);
    RowData actual = deserializationSchema.deserialize("abc\t123\tfalse".getBytes());
    assertEquals(expected, actual);
}

Example 45 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class DataGenTableSourceFactoryTest method runGenerator.

private List<RowData> runGenerator(ResolvedSchema schema, DescriptorProperties descriptor) throws Exception {
    DynamicTableSource source = createTableSource(schema, descriptor.asMap());
    assertTrue(source instanceof DataGenTableSource);
    DataGenTableSource dataGenTableSource = (DataGenTableSource) source;
    DataGeneratorSource<RowData> gen = dataGenTableSource.createSource();
    // test java serialization.
    gen = InstantiationUtil.clone(gen);
    StreamSource<RowData, DataGeneratorSource<RowData>> src = new StreamSource<>(gen);
    AbstractStreamOperatorTestHarness<RowData> testHarness = new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
    testHarness.open();
    TestContext ctx = new TestContext();
    gen.run(ctx);
    return ctx.results;
}