Examples with HoodieMultiTableDeltaStreamer org.apache.hudi.utilities.deltastreamer.HoodieMultiTableDeltaStreamer used on opensource projects

Example 6 with HoodieMultiTableDeltaStreamer

use of org.apache.hudi.utilities.deltastreamer.HoodieMultiTableDeltaStreamer in project hudi by apache.

the class TestHoodieMultiTableDeltaStreamer method testCustomConfigProps.

@Test
public void testCustomConfigProps() throws IOException {
    HoodieMultiTableDeltaStreamer.Config cfg = TestHelpers.getConfig(PROPS_FILENAME_TEST_SOURCE1, dfsBasePath + "/config", TestDataSource.class.getName(), false, false, SchemaRegistryProvider.class);
    HoodieMultiTableDeltaStreamer streamer = new HoodieMultiTableDeltaStreamer(cfg, jsc);
    TableExecutionContext executionContext = streamer.getTableExecutionContexts().get(1);
    assertEquals(2, streamer.getTableExecutionContexts().size());
    assertEquals(dfsBasePath + "/multi_table_dataset/uber_db/dummy_table_uber", executionContext.getConfig().targetBasePath);
    assertEquals("uber_db.dummy_table_uber", executionContext.getConfig().targetTableName);
    assertEquals("topic1", executionContext.getProperties().getString(HoodieMultiTableDeltaStreamer.Constants.KAFKA_TOPIC_PROP));
    assertEquals("_row_key", executionContext.getProperties().getString(DataSourceWriteOptions.RECORDKEY_FIELD().key()));
    assertEquals(TestHoodieDeltaStreamer.TestGenerator.class.getName(), executionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key()));
    assertEquals("uber_hive_dummy_table", executionContext.getProperties().getString(HoodieMultiTableDeltaStreamer.Constants.HIVE_SYNC_TABLE_PROP));
    assertEquals("http://localhost:8081/subjects/random-value/versions/latest", executionContext.getProperties().getString(SchemaRegistryProvider.Config.SRC_SCHEMA_REGISTRY_URL_PROP));
    assertEquals("http://localhost:8081/subjects/topic2-value/versions/latest", streamer.getTableExecutionContexts().get(0).getProperties().getString(SchemaRegistryProvider.Config.SRC_SCHEMA_REGISTRY_URL_PROP));
}

Example 7 with HoodieMultiTableDeltaStreamer

use of org.apache.hudi.utilities.deltastreamer.HoodieMultiTableDeltaStreamer in project hudi by apache.

the class TestHoodieMultiTableDeltaStreamer method testTableLevelProperties.

@Test
public void testTableLevelProperties() throws IOException {
    HoodieMultiTableDeltaStreamer.Config cfg = TestHelpers.getConfig(PROPS_FILENAME_TEST_SOURCE1, dfsBasePath + "/config", TestDataSource.class.getName(), false, false, null);
    HoodieMultiTableDeltaStreamer streamer = new HoodieMultiTableDeltaStreamer(cfg, jsc);
    List<TableExecutionContext> tableExecutionContexts = streamer.getTableExecutionContexts();
    tableExecutionContexts.forEach(tableExecutionContext -> {
        switch(tableExecutionContext.getTableName()) {
            case "dummy_table_short_trip":
                String tableLevelKeyGeneratorClass = tableExecutionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key());
                assertEquals(TestHoodieDeltaStreamer.TestTableLevelGenerator.class.getName(), tableLevelKeyGeneratorClass);
                break;
            default:
                String defaultKeyGeneratorClass = tableExecutionContext.getProperties().getString(DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME().key());
                assertEquals(TestHoodieDeltaStreamer.TestGenerator.class.getName(), defaultKeyGeneratorClass);
        }
    });
}

Example 8 with HoodieMultiTableDeltaStreamer

use of org.apache.hudi.utilities.deltastreamer.HoodieMultiTableDeltaStreamer in project hudi by apache.

the class TestHoodieMultiTableDeltaStreamer method testInvalidPropsFilePath.

@Test
public void testInvalidPropsFilePath() throws IOException {
    HoodieMultiTableDeltaStreamer.Config cfg = TestHelpers.getConfig(PROPS_INVALID_FILE, dfsBasePath + "/config", TestDataSource.class.getName(), true, true, null);
    Exception e = assertThrows(IllegalArgumentException.class, () -> {
        new HoodieMultiTableDeltaStreamer(cfg, jsc);
    }, "Should fail when invalid props file is provided");
    log.debug("Expected error when creating table execution objects", e);
    assertTrue(e.getMessage().contains("Please provide valid common config file path!"));
}