Search in sources :

Example 1 with DFSPropertiesConfiguration

use of org.apache.hudi.common.config.DFSPropertiesConfiguration in project hudi by apache.

the class TestHoodieDeltaStreamer method testKafkaConnectCheckpointProvider.

@Test
public void testKafkaConnectCheckpointProvider() throws IOException {
    String tableBasePath = dfsBasePath + "/test_table";
    String bootstrapPath = dfsBasePath + "/kafka_topic1";
    String partitionPath = bootstrapPath + "/year=2016/month=05/day=01";
    String filePath = partitionPath + "/kafka_topic1+0+100+200.parquet";
    String checkpointProviderClass = "org.apache.hudi.utilities.checkpointing.KafkaConnectHdfsProvider";
    HoodieDeltaStreamer.Config cfg = TestHelpers.makeDropAllConfig(tableBasePath, WriteOperationType.UPSERT);
    TypedProperties props = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/" + PROPS_FILENAME_TEST_SOURCE)).getProps();
    props.put("hoodie.deltastreamer.checkpoint.provider.path", bootstrapPath);
    cfg.initialCheckpointProvider = checkpointProviderClass;
    // create regular kafka connect hdfs dirs
    dfs.mkdirs(new Path(bootstrapPath));
    dfs.mkdirs(new Path(partitionPath));
    // generate parquet files using kafka connect naming convention
    HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator();
    Helpers.saveParquetToDFS(Helpers.toGenericRecords(dataGenerator.generateInserts("000", 100)), new Path(filePath));
    HoodieDeltaStreamer deltaStreamer = new HoodieDeltaStreamer(cfg, jsc, dfs, hdfsTestService.getHadoopConf(), Option.ofNullable(props));
    assertEquals("kafka_topic1,0:200", deltaStreamer.getConfig().checkpoint);
}
Also used : Path(org.apache.hadoop.fs.Path) HoodieDeltaStreamer(org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer) TypedProperties(org.apache.hudi.common.config.TypedProperties) DFSPropertiesConfiguration(org.apache.hudi.common.config.DFSPropertiesConfiguration) HoodieTestDataGenerator(org.apache.hudi.common.testutils.HoodieTestDataGenerator) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 2 with DFSPropertiesConfiguration

use of org.apache.hudi.common.config.DFSPropertiesConfiguration in project hudi by apache.

the class TestHoodieDeltaStreamer method testProps.

@Test
public void testProps() {
    TypedProperties props = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/" + PROPS_FILENAME_TEST_SOURCE)).getProps();
    assertEquals(2, props.getInteger("hoodie.upsert.shuffle.parallelism"));
    assertEquals("_row_key", props.getString("hoodie.datasource.write.recordkey.field"));
    assertEquals("org.apache.hudi.utilities.functional.TestHoodieDeltaStreamer$TestGenerator", props.getString("hoodie.datasource.write.keygenerator.class"));
}
Also used : Path(org.apache.hadoop.fs.Path) TypedProperties(org.apache.hudi.common.config.TypedProperties) DFSPropertiesConfiguration(org.apache.hudi.common.config.DFSPropertiesConfiguration) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 3 with DFSPropertiesConfiguration

use of org.apache.hudi.common.config.DFSPropertiesConfiguration in project hudi by apache.

the class TestDFSPropertiesConfiguration method testIncludes.

@Test
public void testIncludes() {
    DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/t3.props"));
    TypedProperties props = cfg.getProps();
    assertEquals(123, props.getInteger("int.prop"));
    assertEquals(243.4, props.getDouble("double.prop"), 0.001);
    assertTrue(props.getBoolean("boolean.prop"));
    assertEquals("t3.value", props.getString("string.prop"));
    assertEquals(1354354354, props.getLong("long.prop"));
    assertThrows(IllegalStateException.class, () -> {
        cfg.addPropsFromFile(new Path(dfsBasePath + "/t4.props"));
    }, "Should error out on a self-included file.");
}
Also used : Path(org.apache.hadoop.fs.Path) DFSPropertiesConfiguration(org.apache.hudi.common.config.DFSPropertiesConfiguration) TypedProperties(org.apache.hudi.common.config.TypedProperties) Test(org.junit.jupiter.api.Test)

Example 4 with DFSPropertiesConfiguration

use of org.apache.hudi.common.config.DFSPropertiesConfiguration in project hudi by apache.

the class TestDFSPropertiesConfiguration method testLocalFileSystemLoading.

@Test
public void testLocalFileSystemLoading() throws IOException {
    DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/t1.props"));
    cfg.addPropsFromFile(new Path(String.format("file:%s", getClass().getClassLoader().getResource("props/test.properties").getPath())));
    TypedProperties props = cfg.getProps();
    assertEquals(123, props.getInteger("int.prop"));
    assertEquals(113.4, props.getDouble("double.prop"), 0.001);
    assertTrue(props.getBoolean("boolean.prop"));
    assertEquals("str", props.getString("string.prop"));
    assertEquals(1354354354, props.getLong("long.prop"));
    assertEquals(123, props.getInteger("some.random.prop"));
}
Also used : Path(org.apache.hadoop.fs.Path) DFSPropertiesConfiguration(org.apache.hudi.common.config.DFSPropertiesConfiguration) TypedProperties(org.apache.hudi.common.config.TypedProperties) Test(org.junit.jupiter.api.Test)

Example 5 with DFSPropertiesConfiguration

use of org.apache.hudi.common.config.DFSPropertiesConfiguration in project hudi by apache.

the class TestDFSPropertiesConfiguration method testParsing.

@Test
public void testParsing() {
    DFSPropertiesConfiguration cfg = new DFSPropertiesConfiguration(dfs.getConf(), new Path(dfsBasePath + "/t1.props"));
    TypedProperties props = cfg.getProps();
    assertEquals(5, props.size());
    assertThrows(IllegalArgumentException.class, () -> {
        props.getString("invalid.key");
    }, "Should error out here.");
    assertEquals(123, props.getInteger("int.prop"));
    assertEquals(113.4, props.getDouble("double.prop"), 0.001);
    assertTrue(props.getBoolean("boolean.prop"));
    assertEquals("str", props.getString("string.prop"));
    assertEquals(1354354354, props.getLong("long.prop"));
    assertEquals(123, props.getInteger("int.prop", 456));
    assertEquals(113.4, props.getDouble("double.prop", 223.4), 0.001);
    assertTrue(props.getBoolean("boolean.prop", false));
    assertEquals("str", props.getString("string.prop", "default"));
    assertEquals(1354354354, props.getLong("long.prop", 8578494434L));
    assertEquals(456, props.getInteger("bad.int.prop", 456));
    assertEquals(223.4, props.getDouble("bad.double.prop", 223.4), 0.001);
    assertFalse(props.getBoolean("bad.boolean.prop", false));
    assertEquals("default", props.getString("bad.string.prop", "default"));
    assertEquals(8578494434L, props.getLong("bad.long.prop", 8578494434L));
}
Also used : Path(org.apache.hadoop.fs.Path) DFSPropertiesConfiguration(org.apache.hudi.common.config.DFSPropertiesConfiguration) TypedProperties(org.apache.hudi.common.config.TypedProperties) Test(org.junit.jupiter.api.Test)

Aggregations

DFSPropertiesConfiguration (org.apache.hudi.common.config.DFSPropertiesConfiguration)8 Path (org.apache.hadoop.fs.Path)5 TypedProperties (org.apache.hudi.common.config.TypedProperties)5 Test (org.junit.jupiter.api.Test)5 BufferedReader (java.io.BufferedReader)3 IOException (java.io.IOException)3 StringReader (java.io.StringReader)3 HoodieIOException (org.apache.hudi.exception.HoodieIOException)3 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)2 HoodieTestDataGenerator (org.apache.hudi.common.testutils.HoodieTestDataGenerator)1 HoodieDeltaStreamer (org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer)1