Examples with HdfsColMeta com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta used on opensource projects

Example 1 with HdfsColMeta

use of com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta in project plugins by qlangtech.

the class TestDataXHudiWriter method createDataXWriter.

private static HudiTest createDataXWriter(Optional<FileSystemFactory> fsFactory) {
    final DefaultSparkConnGetter sparkConnGetter = new DefaultSparkConnGetter();
    sparkConnGetter.name = "default";
    sparkConnGetter.master = "spark://sparkmaster:7077";
    DataXHudiWriter writer = new DataXHudiWriter() {

        @Override
        public Class<?> getOwnerClass() {
            return DataXHudiWriter.class;
        }

        @Override
        public IHiveConnGetter getHiveConnMeta() {
            return HdfsFileSystemFactoryTestUtils.createHiveConnGetter();
        }

        @Override
        public ISparkConnGetter getSparkConnGetter() {
            return sparkConnGetter;
        }

        @Override
        public FileSystemFactory getFs() {
            return fsFactory.isPresent() ? fsFactory.get() : HdfsFileSystemFactoryTestUtils.getFileSystemFactory();
        }
    };
    writer.template = DataXHudiWriter.getDftTemplate();
    writer.fsName = HdfsFileSystemFactoryTestUtils.FS_NAME;
    writer.setKey(new KeyedPluginStore.Key(null, HdfsFileSystemFactoryTestUtils.testDataXName.getName(), null));
    writer.tabType = HudiWriteTabType.COW.getValue();
    writer.batchOp = BatchOpMode.BULK_INSERT.getValue();
    writer.shuffleParallelism = 3;
    writer.partitionedBy = "pt";
    // writer.batchByteSize = 3456;
    // writer.batchSize = 9527;
    // writer.dbName = dbName;
    writer.writeMode = "insert";
    // writer.autoCreateTable = true;
    // writer.postSql = "drop table @table";
    // writer.preSql = "drop table @table";
    // writer.dataXName = HdfsFileSystemFactoryTestUtils.testDataXName.getName();
    // writer.dbName = dbName;
    // HudiSelectedTab hudiTab = new HudiSelectedTab() {
    // @Override
    // public List<ColMeta> getCols() {
    // return WriterTemplate.createColMetas();
    // }
    // };
    // //hudiTab.partitionPathField = WriterTemplate.kind;
    // hudiTab.recordField = WriterTemplate.customerregisterId;
    // hudiTab.sourceOrderingField = WriterTemplate.lastVer;
    // hudiTab.setWhere("1=1");
    // hudiTab.name = WriterTemplate.TAB_customer_order_relation;
    List<HdfsColMeta> colsMeta = HdfsColMeta.getColsMeta(Configuration.from(IOUtils.loadResourceFromClasspath(writer.getClass(), hudi_datax_writer_assert_without_optional)).getConfiguration(cfgPathParameter));
    HudiSelectedTab tab = new HudiSelectedTab() {

        @Override
        public List<ColMeta> getCols() {
            return colsMeta.stream().map((c) -> {
                ColMeta col = new ColMeta();
                col.setName(c.getName());
                col.setPk(c.pk);
                col.setType(c.type);
                col.setNullable(c.nullable);
                return col;
            }).collect(Collectors.toList());
        }
    };
    tab.name = WriterTemplate.TAB_customer_order_relation;
    tab.partition = new OffPartition();
    tab.sourceOrderingField = "last_ver";
    tab.recordField = "customerregister_id";
    return new HudiTest(writer, WriterTemplate.createCustomer_order_relationTableMap(Optional.of(tab)), tab);
}

Example 2 with HdfsColMeta

use of com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta in project plugins by qlangtech.

the class TestDataXHudiWriterTask method testCsvWrite.

@Test
public void testCsvWrite() throws Exception {
    DefaultRecord[] record = new DefaultRecord[1];
    ByteArrayOutputStream output = new ByteArrayOutputStream();
    TisDataXHudiWriter.Task task = new TisDataXHudiWriter.Task() {

        @Override
        public void init() {
            this.fileType = "csv";
            this.writerSliceConfig = Configuration.from(IOUtils.loadResourceFromClasspath(TestDataXHudiWriterTask.class, TestDataXHudiWriter.hudi_datax_writer_assert_without_optional)).getConfiguration("parameter");
            List<HdfsColMeta> colsMeta = HdfsColMeta.getColsMeta(this.writerSliceConfig);
            record[0] = new DefaultRecord();
            for (HdfsColMeta col : colsMeta) {
                // col.csvType
                switch(col.csvType) {
                    case STRING:
                        record[0].addColumn(new StringColumn("{\"name\":\"" + RandomStringUtils.randomAlphanumeric(4) + "\"}"));
                        break;
                    case BOOLEAN:
                        break;
                    case NUMBER:
                        record[0].addColumn(new LongColumn((long) (Math.random() * 1000)));
                        break;
                }
            }
        }

        @Override
        public void prepare() {
            super.prepare();
        }

        @Override
        protected OutputStream getOutputStream(Path targetPath) {
            return output;
        }
    };
    RecordReceiver records = new RecordReceiver() {

        int index = 0;

        @Override
        public Record getFromReader() {
            if (index++ < 1) {
                return record[0];
            }
            return null;
        }

        @Override
        public void shutdown() {
        }
    };
    task.init();
    task.prepare();
    task.startWrite(records);
    System.out.println(new String(output.toByteArray()));
}

Example 3 with HdfsColMeta

use of com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta in project plugins by qlangtech.

the class CUDCDCTestSuit method startTest.

public void startTest(MQListenerFactory cdcFactory, String tabName) throws Exception {
    BasicDataXRdbmsReader dataxReader = createDataxReader(dataxName, tabName);
    // replay();
    List<SelectedTab> selectedTabs = dataxReader.getSelectedTabs();
    Optional<SelectedTab> firstSelectedTab = selectedTabs.stream().filter((t) -> tabName.equals(t.name)).findFirst();
    Assert.assertTrue("firstSelectedTab:" + tabName + " must be present", firstSelectedTab.isPresent());
    ISelectedTab tab = firstSelectedTab.get();
    this.cols = Lists.newArrayList();
    HdfsColMeta cMeta = null;
    for (ISelectedTab.ColMeta c : tab.getCols()) {
        cMeta = new HdfsColMeta(c.getName(), c.isNullable(), c.isPk(), c.getType());
        cols.add(cMeta);
    }
    IResultRows consumerHandle = getTestBasicFlinkSourceHandle(tabName);
    cdcFactory.setConsumerHandle(consumerHandle.getConsumerHandle());
    IMQListener<JobExecutionResult> imqListener = cdcFactory.create();
    this.verfiyTableCrudProcess(tabName, dataxReader, tab, consumerHandle, imqListener);
    consumerHandle.cancel();
}

Example 4 with HdfsColMeta

use of com.alibaba.datax.plugin.writer.hdfswriter.HdfsColMeta in project plugins by qlangtech.

the class TestRow method getValsList.

public List<String> getValsList(Optional<RowKind> updateVal, List<HdfsColMeta> keys, ValProcessor processor) throws Exception {
    RowKind rowKind = updateVal.isPresent() ? updateVal.get() : this.kind;
    List<String> valsEnum = Lists.newArrayList(rowKind.shortString());
    for (HdfsColMeta key : keys) {
        Object val = null;
        if (rowKind != RowKind.INSERT) {
            RowValsUpdate.UpdatedColVal uptColVal = (RowValsUpdate.UpdatedColVal) updateVals.getObj(key.getName());
            if (uptColVal != null) {
                val = uptColVal.updatedVal;
            }
        }
        if (val == null) {
            val = vals.getObj(key.getName());
        }
        valsEnum.add(key.getName() + ":" + processor.process(vals, key.getName(), val));
    }
    return valsEnum;
}