Examples with ConnectConfig org.apache.rocketmq.connect.runtime.config.ConnectConfig used on opensource projects

Example 1 with ConnectConfig

use of org.apache.rocketmq.connect.runtime.config.ConnectConfig in project rocketmq-externals by apache.

the class ConfigManagementServiceImplTest method init.

@Before
public void init() throws Exception {
    String consumerGroup = UUID.randomUUID().toString();
    String producerGroup = UUID.randomUUID().toString();
    connectConfig = new ConnectConfig();
    connectConfig.setHttpPort(8081);
    connectConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "testConnectorStore");
    connectConfig.setRmqConsumerGroup("testConsumerGroup");
    connectorName = "testConnectorName";
    connectConfig.setRmqConsumerGroup(consumerGroup);
    connectConfig.setRmqProducerGroup(producerGroup);
    connectConfig.setNamesrvAddr("127.0.0.1:9876");
    connectConfig.setRmqMinConsumeThreadNums(1);
    connectConfig.setRmqMaxConsumeThreadNums(32);
    connectConfig.setRmqMessageConsumeTimeout(3 * 1000);
    connectKeyValue = new ConnectKeyValue();
    connectKeyValue.put(RuntimeConfigDefine.CONNECTOR_CLASS, "org.apache.rocketmq.connect.runtime.connectorwrapper.testimpl.TestConnector");
    connectKeyValue.put(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER, "source-record-converter");
    doAnswer(new Answer() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Exception {
            final Message message = invocation.getArgument(0);
            byte[] bytes = message.getBody();
            final Field dataSynchronizerField = ConfigManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
            dataSynchronizerField.setAccessible(true);
            BrokerBasedLog<String, ConnAndTaskConfigs> dataSynchronizer = (BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService);
            final Method decodeKeyValueMethod = BrokerBasedLog.class.getDeclaredMethod("decodeKeyValue", byte[].class);
            decodeKeyValueMethod.setAccessible(true);
            Map<String, ConnAndTaskConfigs> map = (Map<String, ConnAndTaskConfigs>) decodeKeyValueMethod.invoke(dataSynchronizer, bytes);
            final Field dataSynchronizerCallbackField = BrokerBasedLog.class.getDeclaredField("dataSynchronizerCallback");
            dataSynchronizerCallbackField.setAccessible(true);
            final DataSynchronizerCallback<String, ConnAndTaskConfigs> dataSynchronizerCallback = (DataSynchronizerCallback<String, ConnAndTaskConfigs>) dataSynchronizerCallbackField.get(dataSynchronizer);
            for (String key : map.keySet()) {
                dataSynchronizerCallback.onCompletion(null, key, map.get(key));
            }
            return null;
        }
    }).when(producer).send(any(Message.class), any(SendCallback.class));
    configManagementService = new ConfigManagementServiceImpl(connectConfig, plugin);
    final Field connectorKeyValueStoreField = ConfigManagementServiceImpl.class.getDeclaredField("connectorKeyValueStore");
    connectorKeyValueStoreField.setAccessible(true);
    connectorKeyValueStore = (KeyValueStore<String, ConnectKeyValue>) connectorKeyValueStoreField.get(configManagementService);
    final Field taskKeyValueStoreField = ConfigManagementServiceImpl.class.getDeclaredField("taskKeyValueStore");
    taskKeyValueStoreField.setAccessible(true);
    taskKeyValueStore = (KeyValueStore<String, List<ConnectKeyValue>>) taskKeyValueStoreField.get(configManagementService);
    final Field dataSynchronizerField = ConfigManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
    dataSynchronizerField.setAccessible(true);
    final Field producerField = BrokerBasedLog.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    producerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService), producer);
    final Field consumerField = BrokerBasedLog.class.getDeclaredField("consumer");
    consumerField.setAccessible(true);
    consumerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService), consumer);
    configManagementService.start();
}

Example 2 with ConnectConfig

use of org.apache.rocketmq.connect.runtime.config.ConnectConfig in project rocketmq-externals by apache.

the class WorkerTest method init.

@Before
public void init() {
    connectConfig = new ConnectConfig();
    connectConfig.setHttpPort(8081);
    connectConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "testConnectorStore");
    connectConfig.setNamesrvAddr("localhost:9876");
    worker = new Worker(connectConfig, positionManagementService, offsetManagementService, plugin);
    Set<WorkerConnector> workingConnectors = new HashSet<>();
    for (int i = 0; i < 3; i++) {
        ConnectKeyValue connectKeyValue = new ConnectKeyValue();
        connectKeyValue.getProperties().put("key1", "TEST-CONN-" + i + "1");
        connectKeyValue.getProperties().put("key2", "TEST-CONN-" + i + "2");
        workingConnectors.add(new WorkerConnector("TEST-CONN-" + i, new TestConnector(), connectKeyValue, connectorContext));
    }
    worker.setWorkingConnectors(workingConnectors);
    assertThat(worker.getWorkingConnectors().size()).isEqualTo(3);
    Set<Runnable> runnables = new HashSet<>();
    for (int i = 0; i < 3; i++) {
        ConnectKeyValue connectKeyValue = new ConnectKeyValue();
        connectKeyValue.getProperties().put("key1", "TEST-TASK-" + i + "1");
        connectKeyValue.getProperties().put("key2", "TEST-TASK-" + i + "2");
        runnables.add(new WorkerSourceTask("TEST-CONN-" + i, new TestSourceTask(), connectKeyValue, new TestPositionManageServiceImpl(), new TestConverter(), producer, new AtomicReference(WorkerState.STARTED)));
    }
    worker.setWorkingTasks(runnables);
    assertThat(worker.getWorkingTasks().size()).isEqualTo(3);
    worker.start();
}

Example 3 with ConnectConfig

use of org.apache.rocketmq.connect.runtime.config.ConnectConfig in project rocketmq-externals by apache.

the class PositionManagementServiceImplTest method init.

@Before
public void init() throws Exception {
    connectConfig = new ConnectConfig();
    connectConfig.setHttpPort(8081);
    connectConfig.setNamesrvAddr("localhost:9876");
    connectConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "testConnectorStore");
    connectConfig.setRmqConsumerGroup("testConsumerGroup");
    doAnswer(new Answer() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Exception {
            final Message message = invocation.getArgument(0);
            byte[] bytes = message.getBody();
            final Field dataSynchronizerField = PositionManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
            dataSynchronizerField.setAccessible(true);
            BrokerBasedLog<String, Map> dataSynchronizer = (BrokerBasedLog<String, Map>) dataSynchronizerField.get(positionManagementService);
            final Method decodeKeyValueMethod = BrokerBasedLog.class.getDeclaredMethod("decodeKeyValue", byte[].class);
            decodeKeyValueMethod.setAccessible(true);
            Map<String, Map> map = (Map<String, Map>) decodeKeyValueMethod.invoke(dataSynchronizer, bytes);
            final Field dataSynchronizerCallbackField = BrokerBasedLog.class.getDeclaredField("dataSynchronizerCallback");
            dataSynchronizerCallbackField.setAccessible(true);
            final DataSynchronizerCallback<String, Map> dataSynchronizerCallback = (DataSynchronizerCallback<String, Map>) dataSynchronizerCallbackField.get(dataSynchronizer);
            for (String key : map.keySet()) {
                dataSynchronizerCallback.onCompletion(null, key, map.get(key));
            }
            return null;
        }
    }).when(producer).send(any(Message.class), any(SendCallback.class));
    positionManagementService = new PositionManagementServiceImpl(connectConfig);
    final Field dataSynchronizerField = PositionManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
    dataSynchronizerField.setAccessible(true);
    final Field producerField = BrokerBasedLog.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    producerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(positionManagementService), producer);
    final Field consumerField = BrokerBasedLog.class.getDeclaredField("consumer");
    consumerField.setAccessible(true);
    consumerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(positionManagementService), consumer);
    positionManagementService.start();
    Field positionStoreField = PositionManagementServiceImpl.class.getDeclaredField("positionStore");
    positionStoreField.setAccessible(true);
    positionStore = (KeyValueStore<ByteBuffer, ByteBuffer>) positionStoreField.get(positionManagementService);
    Field needSyncPartitionField = PositionManagementServiceImpl.class.getDeclaredField("needSyncPartition");
    needSyncPartitionField.setAccessible(true);
    needSyncPartition = (ConcurrentSet<ByteBuffer>) needSyncPartitionField.get(positionManagementService);
    sourcePartition = ByteBuffer.wrap("127.0.0.13306".getBytes("UTF-8"));
    JSONObject jsonObject = new JSONObject();
    // jsonObject.put(MysqlConstants.BINLOG_FILENAME, "binlogFilename");
    // jsonObject.put(MysqlConstants.NEXT_POSITION, "100");
    sourcePosition = ByteBuffer.wrap(jsonObject.toJSONString().getBytes());
    positions = new HashMap<ByteBuffer, ByteBuffer>() {

        {
            put(sourcePartition, sourcePosition);
        }
    };
}

Example 4 with ConnectConfig

use of org.apache.rocketmq.connect.runtime.config.ConnectConfig in project rocketmq-externals by apache.

the class BrokerBasedLogTest method init.

@Before
public void init() throws IllegalAccessException, NoSuchFieldException {
    topicName = "testTopicName";
    consumerGroup = "testConsumerGroup1";
    producerGroup = "testProducerGroup1";
    connectConfig = new ConnectConfig();
    connectConfig.setRmqConsumerGroup(consumerGroup);
    connectConfig.setRmqProducerGroup(producerGroup);
    connectConfig.setNamesrvAddr("127.0.0.1:9876");
    connectConfig.setRmqMinConsumeThreadNums(1);
    connectConfig.setRmqMaxConsumeThreadNums(32);
    connectConfig.setRmqMessageConsumeTimeout(3 * 1000);
    doReturn(new byte[0]).when(converter).objectToByte(any(Object.class));
    brokerBasedLog = new BrokerBasedLog(connectConfig, topicName, consumerGroup, dataSynchronizerCallback, converter, converter);
    final Field producerField = BrokerBasedLog.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    producerField.set(brokerBasedLog, producer);
    final Field consumerField = BrokerBasedLog.class.getDeclaredField("consumer");
    consumerField.setAccessible(true);
    consumerField.set(brokerBasedLog, consumer);
}

Example 5 with ConnectConfig

use of org.apache.rocketmq.connect.runtime.config.ConnectConfig in project rocketmq-externals by apache.

the class ConnectStartup method createConnectController.

/**
 * Read configs from command line and create connect controller.
 *
 * @param args
 * @return
 */
private static ConnectController createConnectController(String[] args) {
    try {
        // Build the command line options.
        Options options = ServerUtil.buildCommandlineOptions(new Options());
        commandLine = ServerUtil.parseCmdLine("connect", args, buildCommandlineOptions(options), new PosixParser());
        if (null == commandLine) {
            System.exit(-1);
        }
        // Load configs from command line.
        ConnectConfig connectConfig = new ConnectConfig();
        if (commandLine.hasOption('c')) {
            String file = commandLine.getOptionValue('c');
            if (file != null) {
                configFile = file;
                InputStream in = new BufferedInputStream(new FileInputStream(file));
                properties = new Properties();
                properties.load(in);
                FileAndPropertyUtil.properties2Object(properties, connectConfig);
                in.close();
            }
        }
        // Create controller and initialize.
        ConnectController controller = new ConnectController(connectConfig);
        controller.initialize();
        // Invoked when shutdown.
        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {

            private volatile boolean hasShutdown = false;

            private AtomicInteger shutdownTimes = new AtomicInteger(0);

            @Override
            public void run() {
                synchronized (this) {
                    log.info("Shutdown hook was invoked, {}", this.shutdownTimes.incrementAndGet());
                    if (!this.hasShutdown) {
                        this.hasShutdown = true;
                        long beginTime = System.currentTimeMillis();
                        controller.shutdown();
                        long consumingTimeTotal = System.currentTimeMillis() - beginTime;
                        log.info("Shutdown hook over, consuming total time(ms): {}", consumingTimeTotal);
                    }
                }
            }
        }, "ShutdownHook"));
        return controller;
    } catch (Throwable e) {
        e.printStackTrace();
        System.exit(-1);
    }
    return null;
}