Examples with TypedPropertiesBuilder com.alibaba.datax.plugin.writer.hudi.TypedPropertiesBuilder used on opensource projects

Example 1 with TypedPropertiesBuilder

use of com.alibaba.datax.plugin.writer.hudi.TypedPropertiesBuilder in project plugins by qlangtech.

the class HudiDumpPostTask method writeSourceProps.

private void writeSourceProps(ITISFileSystem fs, IPath dumpDir, IPath fsSourcePropsPath) {
    IPath fsSourceSchemaPath = HudiTableMeta.createFsSourceSchema(fs, this.hudiTab.getName(), dumpDir, this.hudiTab);
    IPath tabDumpParentPath = TisDataXHudiWriter.createTabDumpParentPath(fs, dumpDir);
    try (OutputStream write = fs.create(fsSourcePropsPath, true)) {
        // TypedProperties props = new TypedProperties();
        TypedPropertiesBuilder props = new TypedPropertiesBuilder();
        String shuffleParallelism = String.valueOf(this.hudiWriter.shuffleParallelism);
        props.setProperty("hoodie.upsert.shuffle.parallelism", shuffleParallelism);
        props.setProperty("hoodie.insert.shuffle.parallelism", (shuffleParallelism));
        props.setProperty("hoodie.delete.shuffle.parallelism", (shuffleParallelism));
        props.setProperty("hoodie.bulkinsert.shuffle.parallelism", (shuffleParallelism));
        props.setProperty("hoodie.embed.timeline.server", "true");
        props.setProperty("hoodie.filesystem.view.type", "EMBEDDED_KV_STORE");
        // @see HoodieCompactionConfig.INLINE_COMPACT
        // props.setProperty("hoodie.compact.inline", (hudiTabType == HudiWriteTabType.MOR) ? "true" : "false");
        // BasicFSWriter writerPlugin = this.getWriterPlugin();
        // https://spark.apache.org/docs/3.2.1/sql-data-sources-csv.html
        props.setProperty("hoodie.deltastreamer.source.dfs.root", String.valueOf(tabDumpParentPath));
        props.setProperty("hoodie.deltastreamer.csv.header", Boolean.toString(TisDataXHudiWriter.CSV_FILE_USE_HEADER));
        props.setProperty("hoodie.deltastreamer.csv.sep", String.valueOf(TisDataXHudiWriter.CSV_Column_Separator));
        props.setProperty("hoodie.deltastreamer.csv.nullValue", TisDataXHudiWriter.CSV_NULL_VALUE);
        props.setProperty("hoodie.deltastreamer.csv.escape", String.valueOf(TisDataXHudiWriter.CSV_ESCAPE_CHAR));
        // props.setProperty("hoodie.deltastreamer.csv.escapeQuotes", "false");
        props.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.file", String.valueOf(fsSourceSchemaPath));
        props.setProperty("hoodie.deltastreamer.schemaprovider.target.schema.file", String.valueOf(fsSourceSchemaPath));
        // please reference: DataSourceWriteOptions , HiveSyncConfig
        final IHiveConnGetter hiveMeta = this.hudiWriter.getHiveConnMeta();
        props.setProperty("hoodie.datasource.hive_sync.database", hiveMeta.getDbName());
        props.setProperty("hoodie.datasource.hive_sync.table", this.hudiTab.getName());
        if (this.hudiTab.partition == null) {
            throw new IllegalStateException("hudiPlugin.partitionedBy can not be empty");
        }
        this.hudiTab.partition.setProps(props, this.hudiWriter);
        // props.setProperty("hoodie.datasource.hive_sync.partition_fields", hudiPlugin.partitionedBy);
        // // "org.apache.hudi.hive.MultiPartKeysValueExtractor";
        // // partition 分区值抽取类
        // props.setProperty("hoodie.datasource.hive_sync.partition_extractor_class"
        // , "org.apache.hudi.hive.MultiPartKeysValueExtractor");
        Optional<HiveUserToken> hiveUserToken = hiveMeta.getUserToken();
        if (hiveUserToken.isPresent()) {
            HiveUserToken token = hiveUserToken.get();
            props.setProperty("hoodie.datasource.hive_sync.username", token.userName);
            props.setProperty("hoodie.datasource.hive_sync.password", token.password);
        }
        props.setProperty("hoodie.datasource.hive_sync.jdbcurl", hiveMeta.getJdbcUrl());
        props.setProperty("hoodie.datasource.hive_sync.mode", "jdbc");
        props.setProperty("hoodie.datasource.write.recordkey.field", this.hudiTab.recordField);
        // props.setProperty("hoodie.datasource.write.partitionpath.field", hudiWriter.partitionedBy);
        props.store(write);
    } catch (IOException e) {
        throw new RuntimeException("faild to write " + tabDumpParentPath + " CSV file metaData", e);
    }
}