Examples with ISparkConnGetter com.qlangtech.tis.config.spark.ISparkConnGetter used on opensource projects

Example 1 with ISparkConnGetter

use of com.qlangtech.tis.config.spark.ISparkConnGetter in project plugins by qlangtech.

the class TestDataXHudiWriter method createDataXWriter.

private static HudiTest createDataXWriter(Optional<FileSystemFactory> fsFactory) {
    final DefaultSparkConnGetter sparkConnGetter = new DefaultSparkConnGetter();
    sparkConnGetter.name = "default";
    sparkConnGetter.master = "spark://sparkmaster:7077";
    DataXHudiWriter writer = new DataXHudiWriter() {

        @Override
        public Class<?> getOwnerClass() {
            return DataXHudiWriter.class;
        }

        @Override
        public IHiveConnGetter getHiveConnMeta() {
            return HdfsFileSystemFactoryTestUtils.createHiveConnGetter();
        }

        @Override
        public ISparkConnGetter getSparkConnGetter() {
            return sparkConnGetter;
        }

        @Override
        public FileSystemFactory getFs() {
            return fsFactory.isPresent() ? fsFactory.get() : HdfsFileSystemFactoryTestUtils.getFileSystemFactory();
        }
    };
    writer.template = DataXHudiWriter.getDftTemplate();
    writer.fsName = HdfsFileSystemFactoryTestUtils.FS_NAME;
    writer.setKey(new KeyedPluginStore.Key(null, HdfsFileSystemFactoryTestUtils.testDataXName.getName(), null));
    writer.tabType = HudiWriteTabType.COW.getValue();
    writer.batchOp = BatchOpMode.BULK_INSERT.getValue();
    writer.shuffleParallelism = 3;
    writer.partitionedBy = "pt";
    // writer.batchByteSize = 3456;
    // writer.batchSize = 9527;
    // writer.dbName = dbName;
    writer.writeMode = "insert";
    // writer.autoCreateTable = true;
    // writer.postSql = "drop table @table";
    // writer.preSql = "drop table @table";
    // writer.dataXName = HdfsFileSystemFactoryTestUtils.testDataXName.getName();
    // writer.dbName = dbName;
    // HudiSelectedTab hudiTab = new HudiSelectedTab() {
    // @Override
    // public List<ColMeta> getCols() {
    // return WriterTemplate.createColMetas();
    // }
    // };
    // //hudiTab.partitionPathField = WriterTemplate.kind;
    // hudiTab.recordField = WriterTemplate.customerregisterId;
    // hudiTab.sourceOrderingField = WriterTemplate.lastVer;
    // hudiTab.setWhere("1=1");
    // hudiTab.name = WriterTemplate.TAB_customer_order_relation;
    List<HdfsColMeta> colsMeta = HdfsColMeta.getColsMeta(Configuration.from(IOUtils.loadResourceFromClasspath(writer.getClass(), hudi_datax_writer_assert_without_optional)).getConfiguration(cfgPathParameter));
    HudiSelectedTab tab = new HudiSelectedTab() {

        @Override
        public List<ColMeta> getCols() {
            return colsMeta.stream().map((c) -> {
                ColMeta col = new ColMeta();
                col.setName(c.getName());
                col.setPk(c.pk);
                col.setType(c.type);
                col.setNullable(c.nullable);
                return col;
            }).collect(Collectors.toList());
        }
    };
    tab.name = WriterTemplate.TAB_customer_order_relation;
    tab.partition = new OffPartition();
    tab.sourceOrderingField = "last_ver";
    tab.recordField = "customerregister_id";
    return new HudiTest(writer, WriterTemplate.createCustomer_order_relationTableMap(Optional.of(tab)), tab);
}