Examples with MQConsumeException com.qlangtech.tis.async.message.client.consumer.MQConsumeException used on opensource projects

Example 1 with MQConsumeException

use of com.qlangtech.tis.async.message.client.consumer.MQConsumeException in project plugins by qlangtech.

the class FlinkCDCMongoDBSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        DataXMongodbReader mongoReader = (DataXMongodbReader) dataSource;
        MangoDBDataSourceFactory dsFactory = mongoReader.getDsFactory();
        List<ReaderSource> sourceFunctions = Lists.newArrayList();
        MongoDBSource.Builder<DTO> builder = MongoDBSource.<DTO>builder().hosts(dsFactory.address).database(dsFactory.dbName).collection(mongoReader.collectionName).connectionOptions(sourceFactory.connectionOptions).errorsTolerance(sourceFactory.errorsTolerance).username(dsFactory.getUserName()).password(dsFactory.getPassword()).deserializer(new TISDeserializationSchema());
        if (sourceFactory.errorsLogEnable != null) {
            builder.errorsLogEnable(sourceFactory.errorsLogEnable);
        }
        if (sourceFactory.copyExisting != null) {
            builder.copyExisting(sourceFactory.copyExisting);
        }
        if (sourceFactory.copyExistingMaxThreads != null) {
            builder.copyExistingMaxThreads(sourceFactory.copyExistingMaxThreads);
        }
        if (sourceFactory.copyExistingQueueSize != null) {
            builder.copyExistingMaxThreads(sourceFactory.copyExistingQueueSize);
        }
        if (sourceFactory.pollMaxBatchSize != null) {
            builder.copyExistingMaxThreads(sourceFactory.pollMaxBatchSize);
        }
        if (sourceFactory.pollAwaitTimeMillis != null) {
            builder.copyExistingMaxThreads(sourceFactory.pollAwaitTimeMillis);
        }
        if (sourceFactory.heartbeatIntervalMillis != null) {
            builder.copyExistingMaxThreads(sourceFactory.heartbeatIntervalMillis);
        }
        SourceFunction<DTO> source = builder.build();
        // MongoDBSource.<DTO>builder()
        // .hosts(dsFactory.address)
        // .database(dsFactory.dbName)
        // .collection(mongoReader.collectionName)
        // .connectionOptions(sourceFactory.connectionOptions)
        // .errorsTolerance(sourceFactory.errorsTolerance)
        // .errorsLogEnable(sourceFactory.errorsLogEnable)
        // .copyExisting(sourceFactory.copyExisting)
        // .copyExistingPipeline(sourceFactory.copyExistingPipeline)
        // .copyExistingMaxThreads(sourceFactory.copyExistingMaxThreads)
        // .copyExistingQueueSize(sourceFactory.copyExistingQueueSize)
        // .pollMaxBatchSize(sourceFactory.pollMaxBatchSize)
        // .pollAwaitTimeMillis(sourceFactory.pollAwaitTimeMillis)
        // .heartbeatIntervalMillis(sourceFactory.heartbeatIntervalMillis)
        // //.port(dsFactory.port)
        // // .databaseList(dbs.toArray(new String[dbs.size()])) // monitor all tables under inventory database
        // //                              .tableList(tbs.toArray(new String[tbs.size()]))
        // .username(dsFactory.getUserName())
        // .password(dsFactory.getPassword())
        // //                              .startupOptions(sourceFactory.getStartupOptions())
        // //.debeziumProperties(debeziumProperties)
        // .deserializer(new TISDeserializationSchema()) // converts SourceRecord to JSON String
        // .build();
        sourceFunctions.add(new ReaderSource(dsFactory.address + "_" + dsFactory.dbName + "_" + mongoReader.collectionName, source));
        SourceChannel sourceChannel = new SourceChannel(sourceFunctions);
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 2 with MQConsumeException

use of com.qlangtech.tis.async.message.client.consumer.MQConsumeException in project plugins by qlangtech.

the class FlinkCDCMysqlSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        // TabColIndexer colIndexer = new TabColIndexer(tabs);
        // TISDeserializationSchema deserializationSchema
        // = new TISDeserializationSchema(new MySQLSourceValConvert(colIndexer));
        TISDeserializationSchema deserializationSchema = new TISDeserializationSchema();
        BasicDataXRdbmsReader rdbmsReader = (BasicDataXRdbmsReader) dataSource;
        BasicDataSourceFactory dsFactory = (BasicDataSourceFactory) rdbmsReader.getDataSourceFactory();
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(dsFactory, tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            DateTimeConverter.setDatetimeConverters(MySqlDateTimeConverter.class.getName(), debeziumProperties);
            String[] databases = dbs.toArray(new String[dbs.size()]);
            return Collections.singletonList(new ReaderSource(dbHost + ":" + dsFactory.port + ":" + dbs.stream().collect(Collectors.joining("_")), MySqlSource.<DTO>builder().hostname(dbHost).port(dsFactory.port).databaseList(// monitor all tables under inventory database
            databases).tableList(tbs.toArray(new String[tbs.size()])).serverTimeZone(BasicDataSourceFactory.DEFAULT_SERVER_TIME_ZONE.getId()).username(dsFactory.getUserName()).password(dsFactory.getPassword()).startupOptions(sourceFactory.getStartupOptions()).debeziumProperties(debeziumProperties).deserializer(// converts SourceRecord to JSON String
            deserializationSchema).build()));
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 3 with MQConsumeException

use of com.qlangtech.tis.async.message.client.consumer.MQConsumeException in project plugins by qlangtech.

the class ConsumerListenerForRm method start.

/**
 * 启动，需要在bean中初始化
 */
// @Override
public Object start() throws MQConsumeException {
    try {
        // 广播模式采用动态消费组的方式
        if (messageModel.equals(MessageModel.BROADCASTING)) {
            // InetAddress.getLocalHost().getHostAddress().replace(".", "_");
            consumerGroup = consumerGroup + StringUtils.replace(NetUtils.getHost(), ".", "_");
        }
        if (// MessageConfig.checkUnPublishEnv() &&
        suspend)
            return null;
        consumer = new DefaultMQPushConsumer(consumerGroup);
        consumer.setNamesrvAddr(namesrvAddr);
        consumer.setConsumeThreadMin(consumeThreadMin);
        // 订阅指定topic下tags
        // consumer.subscribe(topic, consumerHandle.getSubExpression());
        /**
         * Consumer第一次启动默认从队列尾部开始消费
         * 如果非第一次启动，那么按照上次消费的位置继续消费
         */
        consumer.setConsumeFromWhere(consumeFromWhere);
        if (consumeFromWhere.equals(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP) && !stringIsEmpty(consumeTimestamp)) {
            consumer.setConsumeTimestamp(consumeTimestamp);
        }
        switch(registerConsumeType) {
            case ORDERLY:
                consumer.registerMessageListener(new BaseListenerOrderly());
                break;
            case CONCURRENTLY:
            default:
                consumer.registerMessageListener(new BaseListenerConcurrently());
                break;
        }
        consumer.start();
    // logger.info("ConsumerListenerForRm started!topic:" + topic + ",expression:" + consumerHandle.getSubExpression()
    // + "  consumerGroup:" + consumerGroup + "   namesrvAddr:" + namesrvAddr);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
    return null;
}

Example 4 with MQConsumeException

use of com.qlangtech.tis.async.message.client.consumer.MQConsumeException in project plugins by qlangtech.

the class TestFlinkCDCMySQLSourceFactory method testBinlogConsumeWithDataStreamRegisterInstaneDetailTable.

/**
 * 测试 instancedetail
 *
 * @throws Exception
 */
@Test
public void testBinlogConsumeWithDataStreamRegisterInstaneDetailTable() throws Exception {
    FlinkCDCMySQLSourceFactory mysqlCDCFactory = new FlinkCDCMySQLSourceFactory();
    mysqlCDCFactory.startupOptions = "latest";
    final String tabName = "instancedetail";
    CUDCDCTestSuit cdcTestSuit = new CUDCDCTestSuit() {

        @Override
        protected BasicDataSourceFactory createDataSourceFactory(TargetResName dataxName) {
            return createMySqlDataSourceFactory(dataxName);
        }

        @Override
        protected String getColEscape() {
            return "`";
        }

        @Override
        protected IResultRows createConsumerHandle(String tabName) {
            return new TestTableRegisterFlinkSourceHandle(tabName, cols);
        }

        @Override
        protected void verfiyTableCrudProcess(String tabName, BasicDataXRdbmsReader dataxReader, ISelectedTab tab, IResultRows consumerHandle, IMQListener<JobExecutionResult> imqListener) throws MQConsumeException, InterruptedException {
            // super.verfiyTableCrudProcess(tabName, dataxReader, tab, consumerHandle, imqListener);
            List<ISelectedTab> tabs = Collections.singletonList(tab);
            List<TestRow> exampleRows = Lists.newArrayList();
            exampleRows.add(this.parseTestRow(RowKind.INSERT, TestFlinkCDCMySQLSourceFactory.class, tabName + "/insert1.txt"));
            Assert.assertEquals(1, exampleRows.size());
            imqListener.start(dataxName, dataxReader, tabs, null);
            Thread.sleep(1000);
            CloseableIterator<Row> snapshot = consumerHandle.getRowSnapshot(tabName);
            BasicDataSourceFactory dataSourceFactory = (BasicDataSourceFactory) dataxReader.getDataSourceFactory();
            Assert.assertNotNull("dataSourceFactory can not be null", dataSourceFactory);
            dataSourceFactory.visitFirstConnection((conn) -> {
                startProcessConn(conn);
                for (TestRow t : exampleRows) {
                    RowVals<Object> vals = t.vals;
                    final String insertBase = "insert into " + createTableName(tabName) + "(" + cols.stream().filter((c) -> vals.notNull(c.getName())).map((col) -> getColEscape() + col.getName() + getColEscape()).collect(Collectors.joining(" , ")) + ") " + "values(" + cols.stream().filter((c) -> vals.notNull(c.getName())).map((col) -> "?").collect(Collectors.joining(" , ")) + ")";
                    PreparedStatement statement = conn.prepareStatement(insertBase);
                    AtomicInteger ci = new AtomicInteger();
                    cols.stream().filter((c) -> vals.notNull(c.getName())).forEach((col) -> {
                        col.type.accept(new DataType.TypeVisitor<Void>() {

                            @Override
                            public Void longType(DataType type) {
                                try {
                                    statement.setLong(ci.incrementAndGet(), Long.parseLong(vals.getString(col.getName())));
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void doubleType(DataType type) {
                                try {
                                    statement.setDouble(ci.incrementAndGet(), Double.parseDouble(vals.getString(col.getName())));
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void dateType(DataType type) {
                                try {
                                    statement.setDate(ci.incrementAndGet(), java.sql.Date.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void timestampType(DataType type) {
                                try {
                                    statement.setTimestamp(ci.incrementAndGet(), java.sql.Timestamp.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void bitType(DataType type) {
                                try {
                                    statement.setByte(ci.incrementAndGet(), Byte.parseByte(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void blobType(DataType type) {
                                try {
                                    try (InputStream input = new ByteArrayInputStream(vals.getString(col.getName()).getBytes(TisUTF8.get()))) {
                                        statement.setBlob(ci.incrementAndGet(), input);
                                    }
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void varcharType(DataType type) {
                                try {
                                    statement.setString(ci.incrementAndGet(), (vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void intType(DataType type) {
                                try {
                                    statement.setInt(ci.incrementAndGet(), Integer.parseInt(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void floatType(DataType type) {
                                try {
                                    statement.setFloat(ci.incrementAndGet(), Float.parseFloat(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void decimalType(DataType type) {
                                try {
                                    statement.setBigDecimal(ci.incrementAndGet(), BigDecimal.valueOf(Double.parseDouble(vals.getString(col.getName()))));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void timeType(DataType type) {
                                try {
                                    statement.setTime(ci.incrementAndGet(), java.sql.Time.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void tinyIntType(DataType dataType) {
                                try {
                                    statement.setShort(ci.incrementAndGet(), Short.parseShort(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void smallIntType(DataType dataType) {
                                tinyIntType(dataType);
                                return null;
                            }
                        });
                    });
                    Assert.assertEquals(1, executePreparedStatement(conn, statement));
                    statement.close();
                    sleepForAWhile();
                    System.out.println("wait to show insert rows");
                    waitForSnapshotStarted(snapshot);
                    List<TestRow> rows = fetchRows(snapshot, 1, false);
                    for (TestRow rr : rows) {
                        System.out.println("------------" + rr.get("instance_id"));
                        assertTestRow(tabName, RowKind.INSERT, consumerHandle, t, rr);
                    }
                }
            });
        }
    };
    cdcTestSuit.startTest(mysqlCDCFactory, tabName);
}

Example 5 with MQConsumeException

use of com.qlangtech.tis.async.message.client.consumer.MQConsumeException in project plugins by qlangtech.

the class FlinkCDCOracleSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName channalName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader reader = (BasicDataXRdbmsReader) dataSource;
        BasicDataSourceFactory f = (BasicDataSourceFactory) reader.getDataSourceFactory();
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(f, (tab) -> tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            return dbs.stream().map((databaseName) -> {
                SourceFunction<DTO> sourceFunction = OracleSource.<DTO>builder().hostname(dbHost).debeziumProperties(debeziumProperties).port(f.port).startupOptions(sourceFactory.getStartupOptions()).database(// monitor XE database
                StringUtils.upperCase(f.dbName)).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(f.getUserName()).password(f.getPassword()).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + f.port + "_" + databaseName, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(channalName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}