Examples with DeltaGenerator org.apache.hudi.integ.testsuite.generator.DeltaGenerator used on opensource projects

Example 1 with DeltaGenerator

use of org.apache.hudi.integ.testsuite.generator.DeltaGenerator in project hudi by apache.

the class WriterContext method initContext.

public void initContext(JavaSparkContext jsc) throws HoodieException {
    try {
        this.schemaProvider = UtilHelpers.createSchemaProvider(cfg.schemaProviderClassName, props, jsc);
        String schemaStr = schemaProvider.getSourceSchema().toString();
        this.hoodieTestSuiteWriter = new HoodieTestSuiteWriter(jsc, props, cfg, schemaStr);
        int inputParallelism = cfg.inputParallelism > 0 ? cfg.inputParallelism : jsc.defaultParallelism();
        this.deltaGenerator = new DeltaGenerator(new DFSDeltaConfig(DeltaOutputMode.valueOf(cfg.outputTypeName), DeltaInputType.valueOf(cfg.inputFormatName), new SerializableConfiguration(jsc.hadoopConfiguration()), cfg.inputBasePath, cfg.targetBasePath, schemaStr, cfg.limitFileSize, inputParallelism, cfg.deleteOldInput), jsc, sparkSession, schemaStr, keyGenerator);
        log.info(String.format("Initialized writerContext with: %s", schemaStr));
    } catch (Exception e) {
        throw new HoodieException("Failed to reinitialize writerContext", e);
    }
}