Examples with DataType com.qlangtech.tis.plugin.ds.DataType used on opensource projects

Example 6 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project plugins by qlangtech.

the class TestFlinkCDCMySQLSourceFactory method testBinlogConsumeWithDataStreamRegisterInstaneDetailTable.

/**
 * 测试 instancedetail
 *
 * @throws Exception
 */
@Test
public void testBinlogConsumeWithDataStreamRegisterInstaneDetailTable() throws Exception {
    FlinkCDCMySQLSourceFactory mysqlCDCFactory = new FlinkCDCMySQLSourceFactory();
    mysqlCDCFactory.startupOptions = "latest";
    final String tabName = "instancedetail";
    CUDCDCTestSuit cdcTestSuit = new CUDCDCTestSuit() {

        @Override
        protected BasicDataSourceFactory createDataSourceFactory(TargetResName dataxName) {
            return createMySqlDataSourceFactory(dataxName);
        }

        @Override
        protected String getColEscape() {
            return "`";
        }

        @Override
        protected IResultRows createConsumerHandle(String tabName) {
            return new TestTableRegisterFlinkSourceHandle(tabName, cols);
        }

        @Override
        protected void verfiyTableCrudProcess(String tabName, BasicDataXRdbmsReader dataxReader, ISelectedTab tab, IResultRows consumerHandle, IMQListener<JobExecutionResult> imqListener) throws MQConsumeException, InterruptedException {
            // super.verfiyTableCrudProcess(tabName, dataxReader, tab, consumerHandle, imqListener);
            List<ISelectedTab> tabs = Collections.singletonList(tab);
            List<TestRow> exampleRows = Lists.newArrayList();
            exampleRows.add(this.parseTestRow(RowKind.INSERT, TestFlinkCDCMySQLSourceFactory.class, tabName + "/insert1.txt"));
            Assert.assertEquals(1, exampleRows.size());
            imqListener.start(dataxName, dataxReader, tabs, null);
            Thread.sleep(1000);
            CloseableIterator<Row> snapshot = consumerHandle.getRowSnapshot(tabName);
            BasicDataSourceFactory dataSourceFactory = (BasicDataSourceFactory) dataxReader.getDataSourceFactory();
            Assert.assertNotNull("dataSourceFactory can not be null", dataSourceFactory);
            dataSourceFactory.visitFirstConnection((conn) -> {
                startProcessConn(conn);
                for (TestRow t : exampleRows) {
                    RowVals<Object> vals = t.vals;
                    final String insertBase = "insert into " + createTableName(tabName) + "(" + cols.stream().filter((c) -> vals.notNull(c.getName())).map((col) -> getColEscape() + col.getName() + getColEscape()).collect(Collectors.joining(" , ")) + ") " + "values(" + cols.stream().filter((c) -> vals.notNull(c.getName())).map((col) -> "?").collect(Collectors.joining(" , ")) + ")";
                    PreparedStatement statement = conn.prepareStatement(insertBase);
                    AtomicInteger ci = new AtomicInteger();
                    cols.stream().filter((c) -> vals.notNull(c.getName())).forEach((col) -> {
                        col.type.accept(new DataType.TypeVisitor<Void>() {

                            @Override
                            public Void longType(DataType type) {
                                try {
                                    statement.setLong(ci.incrementAndGet(), Long.parseLong(vals.getString(col.getName())));
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void doubleType(DataType type) {
                                try {
                                    statement.setDouble(ci.incrementAndGet(), Double.parseDouble(vals.getString(col.getName())));
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void dateType(DataType type) {
                                try {
                                    statement.setDate(ci.incrementAndGet(), java.sql.Date.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void timestampType(DataType type) {
                                try {
                                    statement.setTimestamp(ci.incrementAndGet(), java.sql.Timestamp.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void bitType(DataType type) {
                                try {
                                    statement.setByte(ci.incrementAndGet(), Byte.parseByte(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void blobType(DataType type) {
                                try {
                                    try (InputStream input = new ByteArrayInputStream(vals.getString(col.getName()).getBytes(TisUTF8.get()))) {
                                        statement.setBlob(ci.incrementAndGet(), input);
                                    }
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void varcharType(DataType type) {
                                try {
                                    statement.setString(ci.incrementAndGet(), (vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void intType(DataType type) {
                                try {
                                    statement.setInt(ci.incrementAndGet(), Integer.parseInt(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void floatType(DataType type) {
                                try {
                                    statement.setFloat(ci.incrementAndGet(), Float.parseFloat(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void decimalType(DataType type) {
                                try {
                                    statement.setBigDecimal(ci.incrementAndGet(), BigDecimal.valueOf(Double.parseDouble(vals.getString(col.getName()))));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void timeType(DataType type) {
                                try {
                                    statement.setTime(ci.incrementAndGet(), java.sql.Time.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void tinyIntType(DataType dataType) {
                                try {
                                    statement.setShort(ci.incrementAndGet(), Short.parseShort(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void smallIntType(DataType dataType) {
                                tinyIntType(dataType);
                                return null;
                            }
                        });
                    });
                    Assert.assertEquals(1, executePreparedStatement(conn, statement));
                    statement.close();
                    sleepForAWhile();
                    System.out.println("wait to show insert rows");
                    waitForSnapshotStarted(snapshot);
                    List<TestRow> rows = fetchRows(snapshot, 1, false);
                    for (TestRow rr : rows) {
                        System.out.println("------------" + rr.get("instance_id"));
                        assertTestRow(tabName, RowKind.INSERT, consumerHandle, t, rr);
                    }
                }
            });
        }
    };
    cdcTestSuit.startTest(mysqlCDCFactory, tabName);
}

Example 7 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project plugins by qlangtech.

the class ParseColsResult method parseColsCfg.

public static ParseColsResult parseColsCfg(IFieldErrorHandler msgHandler, Context context, String fieldName, String value) {
    ParseColsResult parseOSSColsResult = new ParseColsResult();
    DataXReaderTabMeta tabMeta = new DataXReaderTabMeta();
    parseOSSColsResult.tabMeta = tabMeta;
    DataXColMeta colMeta = null;
    try {
        JSONArray cols = JSONArray.parseArray(value);
        if (cols.size() < 1) {
            msgHandler.addFieldError(context, fieldName, "请填写读取字段列表内容");
            return parseOSSColsResult;
        }
        Object firstElement = null;
        if (cols.size() == 1 && (firstElement = cols.get(0)) != null && "*".equals(String.valueOf(firstElement))) {
            tabMeta.allCols = true;
            return parseOSSColsResult.ok();
        }
        JSONObject col = null;
        String type = null;
        DataType parseType = null;
        Integer index = null;
        String appValue = null;
        for (int i = 0; i < cols.size(); i++) {
            col = cols.getJSONObject(i);
            type = col.getString("type");
            if (StringUtils.isEmpty(type)) {
                msgHandler.addFieldError(context, fieldName, "index为" + i + "的字段列中，属性type不能为空");
                return parseOSSColsResult.faild();
            }
            parseType = DataXReaderColType.parse(type);
            if (parseType == null) {
                msgHandler.addFieldError(context, fieldName, "index为" + i + "的字段列中，属性type必须为:" + DataXReaderColType.toDesc() + "中之一");
                return parseOSSColsResult.faild();
            }
            colMeta = new DataXColMeta(parseType);
            tabMeta.cols.add(colMeta);
            index = col.getInteger("index");
            appValue = col.getString("value");
            if (index == null && appValue == null) {
                msgHandler.addFieldError(context, fieldName, "index为" + i + "的字段列中，index/value必须选择其一");
                return parseOSSColsResult.faild();
            }
            if (index != null) {
                colMeta.index = index;
            }
            if (appValue != null) {
                colMeta.value = appValue;
            }
        }
    } catch (Exception e) {
        logger.error(value, e);
        msgHandler.addFieldError(context, fieldName, "请检查内容格式是否有误:" + e.getMessage());
        return parseOSSColsResult.faild();
    }
    return parseOSSColsResult.ok();
}

Example 8 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project plugins by qlangtech.

the class BasicDorisStarRocksWriter method generateCreateDDL.

@Override
public StringBuffer generateCreateDDL(IDataxProcessor.TableMap tableMapper) {
    if (!this.autoCreateTable) {
        return null;
    }
    // https://doris.apache.org/master/zh-CN/sql-reference/sql-statements/Data%20Definition/CREATE%20TABLE.html#create-table
    final CreateTableSqlBuilder createTableSqlBuilder = new CreateTableSqlBuilder(tableMapper) {

        @Override
        protected void appendExtraColDef(List<ColWrapper> pks) {
        // if (pk != null) {
        // script.append("  PRIMARY KEY (`").append(pk.getName()).append("`)").append("\n");
        // }
        }

        @Override
        protected List<ColWrapper> preProcessCols(List<ColWrapper> pks, List<ISelectedTab.ColMeta> cols) {
            // 将主键排在最前面
            List<ColWrapper> result = Lists.newArrayList(pks);
            cols.stream().filter((c) -> !c.isPk()).forEach((c) -> {
                result.add(createColWrapper(c));
            });
            return result;
        }

        @Override
        protected void appendTabMeta(List<ColWrapper> pks) {
            script.append(" ENGINE=olap").append("\n");
            if (pks.size() > 0) {
                script.append("PRIMARY KEY(").append(pks.stream().map((pk) -> this.colEscapeChar() + pk.getName() + this.colEscapeChar()).collect(Collectors.joining(","))).append(")\n");
            }
            script.append("DISTRIBUTED BY HASH(");
            if (pks.size() > 0) {
                script.append(pks.stream().map((pk) -> this.colEscapeChar() + pk.getName() + this.colEscapeChar()).collect(Collectors.joining(",")));
            } else {
                List<ISelectedTab.ColMeta> cols = this.getCols();
                Optional<ISelectedTab.ColMeta> firstCol = cols.stream().findFirst();
                if (firstCol.isPresent()) {
                    script.append(firstCol.get().getName());
                } else {
                    throw new IllegalStateException("can not find table:" + getCreateTableName() + " any cols");
                }
            }
            script.append(")\n");
            script.append("BUCKETS 10\n");
            script.append("PROPERTIES(\"replication_num\" = \"1\")");
        }

        @Override
        protected ColWrapper createColWrapper(ISelectedTab.ColMeta c) {
            return new ColWrapper(c) {

                @Override
                public String getMapperType() {
                    return convertType(this.meta);
                }

                @Override
                protected void appendExtraConstraint(BlockScriptBuffer ddlScript) {
                    if (this.meta.isPk()) {
                        ddlScript.append(" NOT NULL");
                    }
                }
            };
        }

        protected String convertType(ISelectedTab.ColMeta col) {
            DataType type = col.getType();
            return type.accept(new DataType.TypeVisitor<String>() {

                @Override
                public String longType(DataType type) {
                    return "BIGINT";
                }

                @Override
                public String doubleType(DataType type) {
                    return "DOUBLE";
                }

                @Override
                public String dateType(DataType type) {
                    return "DATE";
                }

                @Override
                public String timestampType(DataType type) {
                    return "DATETIME";
                }

                @Override
                public String bitType(DataType type) {
                    return "TINYINT";
                }

                @Override
                public String blobType(DataType type) {
                    return varcharType(type);
                }

                @Override
                public String varcharType(DataType type) {
                    return "VARCHAR(" + Math.min(type.columnSize, 65000) + ")";
                }

                @Override
                public String intType(DataType type) {
                    return "INT";
                }

                @Override
                public String floatType(DataType type) {
                    return "FLOAT";
                }

                @Override
                public String decimalType(DataType type) {
                    return "DECIMAL(" + type.columnSize + "," + (type.getDecimalDigits() != null ? type.getDecimalDigits() : 0) + ")";
                }
            });
        }
    };
    return createTableSqlBuilder.build();
}

Example 9 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project plugins by qlangtech.

the class HudiTableMeta method createFsSourceSchema.

public static IPath createFsSourceSchema(ITISFileSystem fs, String tabName, IPath tabDumpDir, HudiSelectedTab hudiTabMeta) {
    List<ISelectedTab.ColMeta> colsMetas = hudiTabMeta.getCols();
    if (CollectionUtils.isEmpty(colsMetas)) {
        throw new IllegalStateException("colsMetas of hudiTabMeta can not be empty");
    }
    IPath fsSourceSchemaPath = fs.getPath(tabDumpDir, "meta/schema.avsc");
    try (OutputStream schemaWriter = fs.getOutputStream(fsSourceSchemaPath)) {
        SchemaBuilder.RecordBuilder<Schema> builder = SchemaBuilder.record(tabName);
        // builder.prop("testFiled", LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)));
        SchemaBuilder.FieldAssembler<Schema> fields = builder.fields();
        for (ISelectedTab.ColMeta meta : colsMetas) {
            meta.getType().accept(new DataType.TypeVisitor<Void>() {

                @Override
                public Void longType(DataType type) {
                    if (meta.isNullable()) {
                        fields.optionalLong(meta.getName());
                    } else {
                        fields.requiredLong(meta.getName());
                    }
                    return null;
                }

                @Override
                public Void doubleType(DataType type) {
                    if (meta.isNullable()) {
                        fields.optionalDouble(meta.getName());
                    } else {
                        fields.requiredDouble(meta.getName());
                    }
                    return null;
                }

                @Override
                public Void dateType(DataType type) {
                    Schema schema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT));
                    addNullableSchema(fields, schema, meta);
                    return null;
                }

                @Override
                public Void timestampType(DataType type) {
                    Schema schema = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG));
                    addNullableSchema(fields, schema, meta);
                    return null;
                }

                @Override
                public Void bitType(DataType type) {
                    if (meta.isNullable()) {
                        fields.optionalInt(meta.getName());
                    } else {
                        fields.requiredInt(meta.getName());
                    }
                    return null;
                }

                @Override
                public Void blobType(DataType type) {
                    varcharType(type);
                    return null;
                }

                @Override
                public Void varcharType(DataType type) {
                    if (meta.isNullable()) {
                        // strType.stringDefault(StringUtils.EMPTY);
                        fields.optionalString(meta.getName());
                    } else {
                        // strType.noDefault();
                        fields.requiredString(meta.getName());
                    }
                    return null;
                }

                @Override
                public Void intType(DataType type) {
                    if (meta.isNullable()) {
                        fields.optionalInt(meta.getName());
                    } else {
                        fields.requiredInt(meta.getName());
                    }
                    return null;
                }

                @Override
                public Void floatType(DataType type) {
                    if (meta.isNullable()) {
                        fields.optionalFloat(meta.getName());
                    } else {
                        fields.requiredFloat(meta.getName());
                    }
                    return null;
                }

                @Override
                public Void decimalType(DataType type) {
                    Schema schema = LogicalTypes.decimal(type.columnSize, type.getDecimalDigits()).addToSchema(Schema.create(Schema.Type.DOUBLE));
                    addNullableSchema(fields, schema, meta);
                    return null;
                }

                @Override
                public Void timeType(DataType type) {
                    Schema schema = LogicalTypes.timeMillis().addToSchema(Schema.create(Schema.Type.INT));
                    addNullableSchema(fields, schema, meta);
                    return null;
                }

                @Override
                public Void tinyIntType(DataType dataType) {
                    smallIntType(dataType);
                    return null;
                }

                @Override
                public Void smallIntType(DataType dataType) {
                    if (meta.isNullable()) {
                        fields.optionalInt(meta.getName());
                    } else {
                        fields.requiredInt(meta.getName());
                    }
                    return null;
                }
            });
        // SupportHiveDataType hiveDataType = DataType.convert2HiveType(meta.getType());
        // switch (hiveDataType) {
        // case STRING:
        // case DATE:
        // case TIMESTAMP:
        // case VARCHAR:
        // case CHAR:
        // // fields.nullableString(meta.colName, StringUtils.EMPTY);
        // //                            if (meta.nullable) {
        // //                                fields.nullableString(meta.colName, StringUtils.EMPTY);
        // //                            } else {
        // // fields.requiredString(meta.colName);
        // // SchemaBuilder.StringDefault<Schema> strType = fields.name(meta.colName).type().stringType();
        // if (meta.isNullable()) {
        // // strType.stringDefault(StringUtils.EMPTY);
        // fields.optionalString(meta.getName());
        // } else {
        // //   strType.noDefault();
        // fields.requiredString(meta.getName());
        // }
        // //}
        // break;
        // case DOUBLE:
        // if (meta.isNullable()) {
        // fields.optionalDouble(meta.getName());
        // } else {
        // fields.requiredDouble(meta.getName());
        // }
        // break;
        // case INT:
        // case TINYINT:
        // case SMALLINT:
        // if (meta.isNullable()) {
        // fields.optionalInt(meta.getName());
        // } else {
        // fields.requiredInt(meta.getName());
        // }
        // break;
        // case BOOLEAN:
        // if (meta.isNullable()) {
        // fields.optionalBoolean(meta.getName());
        // } else {
        // fields.requiredBoolean(meta.getName());
        // }
        // break;
        // case BIGINT:
        // if (meta.isNullable()) {
        // fields.optionalLong(meta.getName());
        // } else {
        // fields.requiredLong(meta.getName());
        // }
        // break;
        // case FLOAT:
        // if (meta.isNullable()) {
        // fields.optionalFloat(meta.getName());
        // } else {
        // fields.requiredFloat(meta.getName());
        // }
        // break;
        // default:
        // throw new IllegalStateException("illegal type:" + hiveDataType);
        // }
        }
        Schema schema = fields.endRecord();
        if (schema.getFields().size() != colsMetas.size()) {
            throw new IllegalStateException("schema.getFields():" + schema.getFields().size() + " is not equal to 'colsMeta.size()':" + colsMetas.size());
        }
        IOUtils.write(schema.toString(true), schemaWriter, TisUTF8.get());
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    return fsSourceSchemaPath;
}

Example 10 with DataType

use of com.qlangtech.tis.plugin.ds.DataType in project tis by qlangtech.

the class DataxAction method doSaveWriterColsMeta.

/**
 * @param context
 */
@Func(value = PermissionConstant.DATAX_MANAGE)
public void doSaveWriterColsMeta(Context context) {
    String dataxName = this.getString(PARAM_KEY_DATAX_NAME);
    DataxProcessor.DataXCreateProcessMeta processMeta = DataxProcessor.getDataXCreateProcessMeta(this, dataxName);
    if ((processMeta.isReaderRDBMS())) {
        throw new IllegalStateException("can not process the flow with:" + processMeta.toString());
    }
    List<ISelectedTab.ColMeta> writerCols = Lists.newArrayList();
    IDataxProcessor.TableMap tableMapper = new IDataxProcessor.TableMap(new ISelectedTab() {

        @Override
        public List<ColMeta> getCols() {
            return writerCols;
        }
    });
    // tableMapper.setSourceCols(writerCols);
    // //////////////////
    final String keyColsMeta = "colsMeta";
    IControlMsgHandler handler = new DelegateControl4JsonPostMsgHandler(this, this.parseJsonPost());
    if (!Validator.validate(handler, context, // 
    Validator.fieldsValidator(// 
    "writerTargetTabName", new Validator.FieldValidators(Validator.require, Validator.db_col_name) {

        @Override
        public void setFieldVal(String val) {
            tableMapper.setTo(val);
        }
    }, "writerFromTabName", new Validator.FieldValidators(Validator.require, Validator.db_col_name) {

        @Override
        public void setFieldVal(String val) {
            tableMapper.setFrom(val);
        }
    }, // 
    keyColsMeta, new Validator.FieldValidators(Validator.require) {

        @Override
        public void setFieldVal(String val) {
        }
    }, new Validator.IFieldValidator() {

        @Override
        public boolean validate(IFieldErrorHandler msgHandler, Context context, String fieldKey, String fieldData) {
            ISelectedTab.ColMeta colMeta = null;
            JSONArray targetCols = JSON.parseArray(fieldData);
            JSONObject targetCol = null;
            int index;
            String targetColName = null;
            if (targetCols.size() < 1) {
                msgHandler.addFieldError(context, fieldKey, "Writer目标表列不能为空");
                return false;
            }
            Map<String, Integer> existCols = Maps.newHashMap();
            boolean validateFaild = false;
            Integer previousColIndex = null;
            for (int i = 0; i < targetCols.size(); i++) {
                targetCol = targetCols.getJSONObject(i);
                index = targetCol.getInteger("index");
                targetColName = targetCol.getString("name");
                if (StringUtils.isNotBlank(targetColName) && (previousColIndex = existCols.put(targetColName, index)) != null) {
                    msgHandler.addFieldError(context, keyColsMeta + "[" + previousColIndex + "]", "内容不能与第" + index + "行重复");
                    msgHandler.addFieldError(context, keyColsMeta + "[" + index + "]", "内容不能与第" + previousColIndex + "行重复");
                    return false;
                }
                if (!Validator.require.validate(DataxAction.this, context, keyColsMeta + "[" + index + "]", targetColName)) {
                    validateFaild = true;
                } else if (!Validator.db_col_name.validate(DataxAction.this, context, keyColsMeta + "[" + index + "]", targetColName)) {
                    validateFaild = true;
                }
                colMeta = new ISelectedTab.ColMeta();
                colMeta.setName(targetColName);
                DataType dataType = targetCol.getObject("type", DataType.class);
                // colMeta.setType(ISelectedTab.DataXReaderColType.parse(targetCol.getString("type")));
                colMeta.setType(dataType);
                writerCols.add(colMeta);
            }
            return !validateFaild;
        }
    }))) {
        return;
    }
    this.saveTableMapper(this, dataxName, Collections.singletonList(tableMapper));
}