Examples with HoodieTestSuiteConfig org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.HoodieTestSuiteConfig used on opensource projects

Example 6 with HoodieTestSuiteConfig

use of org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.HoodieTestSuiteConfig in project hudi by apache.

the class TestHoodieTestSuiteJob method testDagWithInsertUpsertAndValidate.

// Tests in this class add to the test build time significantly. Since this is a Integration Test (end to end), we
// would like to run this as a nightly build which is a TODO.
// TODO : Clean up input / result paths after each test
@MethodSource("configParams")
public void testDagWithInsertUpsertAndValidate(boolean useDeltaStreamer, String tableType) throws Exception {
    this.cleanDFSDirs();
    String inputBasePath = dfsBasePath + "/input/" + UUID.randomUUID().toString();
    String outputBasePath = dfsBasePath + "/result/" + UUID.randomUUID().toString();
    HoodieTestSuiteConfig cfg = makeConfig(inputBasePath, outputBasePath, useDeltaStreamer, tableType);
    cfg.workloadDagGenerator = ComplexDagGenerator.class.getName();
    HoodieTestSuiteJob hoodieTestSuiteJob = new HoodieTestSuiteJob(cfg, jsc);
    hoodieTestSuiteJob.runTestSuite();
    HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(new Configuration()).setBasePath(cfg.targetBasePath).build();
    assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().getInstants().count(), 2);
}

Example 7 with HoodieTestSuiteConfig

use of org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.HoodieTestSuiteConfig in project hudi by apache.

the class TestHoodieTestSuiteJob method testMORFullDagFromYaml.

@Test
public void testMORFullDagFromYaml() throws Exception {
    boolean useDeltaStreamer = false;
    this.cleanDFSDirs();
    String inputBasePath = dfsBasePath + "/input";
    String outputBasePath = dfsBasePath + "/result";
    HoodieTestSuiteConfig cfg = makeConfig(inputBasePath, outputBasePath, useDeltaStreamer, HoodieTableType.MERGE_ON_READ.name());
    cfg.workloadYamlPath = dfsBasePath + "/" + MOR_DAG_FILE_NAME;
    HoodieTestSuiteJob hoodieTestSuiteJob = new HoodieTestSuiteJob(cfg, jsc);
    hoodieTestSuiteJob.runTestSuite();
    HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(new Configuration()).setBasePath(cfg.targetBasePath).build();
// assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().getInstants().count(), 7);
}