Examples with ConnAndTaskConfigs org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs used on opensource projects

Example 1 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class RebalanceImpl method doRebalance.

/**
 * Distribute connectors and tasks according to the {@link RebalanceImpl#allocateConnAndTaskStrategy}.
 */
public void doRebalance() {
    List<String> curAliveWorkers = clusterManagementService.getAllAliveWorkers();
    log.info("Current Alive workers : " + curAliveWorkers.size());
    Map<String, ConnectKeyValue> curConnectorConfigs = configManagementService.getConnectorConfigs();
    log.info("Current ConnectorConfigs : " + curConnectorConfigs);
    Map<String, List<ConnectKeyValue>> curTaskConfigs = configManagementService.getTaskConfigs();
    log.info("Current TaskConfigs : " + curTaskConfigs);
    ConnAndTaskConfigs allocateResult = allocateConnAndTaskStrategy.allocate(curAliveWorkers, clusterManagementService.getCurrentWorker(), curConnectorConfigs, curTaskConfigs);
    log.info("Allocated connector:{}", allocateResult.getConnectorConfigs());
    log.info("Allocated task:{}", allocateResult.getTaskConfigs());
    updateProcessConfigsInRebalance(allocateResult);
}

Example 2 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class DefaultAllocateConnAndTaskStrategy method allocate.

@Override
public ConnAndTaskConfigs allocate(List<String> allWorker, String curWorker, Map<String, ConnectKeyValue> connectorConfigs, Map<String, List<ConnectKeyValue>> taskConfigs) {
    ConnAndTaskConfigs allocateResult = new ConnAndTaskConfigs();
    if (null == allWorker || 0 == allWorker.size()) {
        return allocateResult;
    }
    List<String> sortedWorkers = new ArrayList<>(allWorker);
    Collections.sort(sortedWorkers);
    log.debug("sortedWorkers: {}", sortedWorkers);
    Map<String, ConnectKeyValue> sortedConnectorConfigs = getSortedMap(connectorConfigs);
    log.debug("SortedConnectorConfigs: {}", sortedConnectorConfigs);
    Map<String, List<ConnectKeyValue>> sortedTaskConfigs = getSortedMap(taskConfigs);
    log.debug("SortedTaskConfigs: {}", sortedTaskConfigs);
    int index = 0;
    for (String connectorName : sortedConnectorConfigs.keySet()) {
        String allocatedWorker = sortedWorkers.get(index % sortedWorkers.size());
        index++;
        if (!curWorker.equals(allocatedWorker)) {
            continue;
        }
        allocateResult.getConnectorConfigs().put(connectorName, sortedConnectorConfigs.get(connectorName));
    }
    for (String connectorName : sortedTaskConfigs.keySet()) {
        for (ConnectKeyValue keyValue : sortedTaskConfigs.get(connectorName)) {
            String allocatedWorker = sortedWorkers.get(index % sortedWorkers.size());
            index++;
            if (!curWorker.equals(allocatedWorker)) {
                continue;
            }
            if (null == allocateResult.getTaskConfigs().get(connectorName)) {
                allocateResult.getTaskConfigs().put(connectorName, new ArrayList<>());
            }
            allocateResult.getTaskConfigs().get(connectorName).add(keyValue);
        }
    }
    return allocateResult;
}

Example 3 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class ConfigManagementServiceImplTest method init.

@Before
public void init() throws Exception {
    String consumerGroup = UUID.randomUUID().toString();
    String producerGroup = UUID.randomUUID().toString();
    connectConfig = new ConnectConfig();
    connectConfig.setHttpPort(8081);
    connectConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "testConnectorStore");
    connectConfig.setRmqConsumerGroup("testConsumerGroup");
    connectorName = "testConnectorName";
    connectConfig.setRmqConsumerGroup(consumerGroup);
    connectConfig.setRmqProducerGroup(producerGroup);
    connectConfig.setNamesrvAddr("127.0.0.1:9876");
    connectConfig.setRmqMinConsumeThreadNums(1);
    connectConfig.setRmqMaxConsumeThreadNums(32);
    connectConfig.setRmqMessageConsumeTimeout(3 * 1000);
    connectKeyValue = new ConnectKeyValue();
    connectKeyValue.put(RuntimeConfigDefine.CONNECTOR_CLASS, "org.apache.rocketmq.connect.runtime.connectorwrapper.testimpl.TestConnector");
    connectKeyValue.put(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER, "source-record-converter");
    doAnswer(new Answer() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Exception {
            final Message message = invocation.getArgument(0);
            byte[] bytes = message.getBody();
            final Field dataSynchronizerField = ConfigManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
            dataSynchronizerField.setAccessible(true);
            BrokerBasedLog<String, ConnAndTaskConfigs> dataSynchronizer = (BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService);
            final Method decodeKeyValueMethod = BrokerBasedLog.class.getDeclaredMethod("decodeKeyValue", byte[].class);
            decodeKeyValueMethod.setAccessible(true);
            Map<String, ConnAndTaskConfigs> map = (Map<String, ConnAndTaskConfigs>) decodeKeyValueMethod.invoke(dataSynchronizer, bytes);
            final Field dataSynchronizerCallbackField = BrokerBasedLog.class.getDeclaredField("dataSynchronizerCallback");
            dataSynchronizerCallbackField.setAccessible(true);
            final DataSynchronizerCallback<String, ConnAndTaskConfigs> dataSynchronizerCallback = (DataSynchronizerCallback<String, ConnAndTaskConfigs>) dataSynchronizerCallbackField.get(dataSynchronizer);
            for (String key : map.keySet()) {
                dataSynchronizerCallback.onCompletion(null, key, map.get(key));
            }
            return null;
        }
    }).when(producer).send(any(Message.class), any(SendCallback.class));
    configManagementService = new ConfigManagementServiceImpl(connectConfig, plugin);
    final Field connectorKeyValueStoreField = ConfigManagementServiceImpl.class.getDeclaredField("connectorKeyValueStore");
    connectorKeyValueStoreField.setAccessible(true);
    connectorKeyValueStore = (KeyValueStore<String, ConnectKeyValue>) connectorKeyValueStoreField.get(configManagementService);
    final Field taskKeyValueStoreField = ConfigManagementServiceImpl.class.getDeclaredField("taskKeyValueStore");
    taskKeyValueStoreField.setAccessible(true);
    taskKeyValueStore = (KeyValueStore<String, List<ConnectKeyValue>>) taskKeyValueStoreField.get(configManagementService);
    final Field dataSynchronizerField = ConfigManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
    dataSynchronizerField.setAccessible(true);
    final Field producerField = BrokerBasedLog.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    producerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService), producer);
    final Field consumerField = BrokerBasedLog.class.getDeclaredField("consumer");
    consumerField.setAccessible(true);
    consumerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService), consumer);
    configManagementService.start();
}

Example 4 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class DefaultAllocateConnAndTaskStrategyTest method testAllocate.

@Test
public void testAllocate() {
    DefaultAllocateConnAndTaskStrategy defaultAllocateConnAndTaskStrategy = new DefaultAllocateConnAndTaskStrategy();
    List<String> allWorker = new ArrayList<String>() {

        {
            add("workId1");
            add("workId2");
            add("workId3");
        }
    };
    String curWorker = "workId3";
    Map<String, ConnectKeyValue> connectorConfigs = new HashMap<String, ConnectKeyValue>() {

        {
            put("connectorConfig1", new ConnectKeyValue());
            put("connectorConfig2", new ConnectKeyValue());
            put("connectorConfig3", new ConnectKeyValue());
        }
    };
    ConnectKeyValue connectKeyValue1 = new ConnectKeyValue();
    ConnectKeyValue connectKeyValue2 = new ConnectKeyValue();
    ConnectKeyValue connectKeyValue3 = new ConnectKeyValue();
    ConnectKeyValue connectKeyValue4 = new ConnectKeyValue();
    ConnectKeyValue connectKeyValue5 = new ConnectKeyValue();
    ConnectKeyValue connectKeyValue6 = new ConnectKeyValue();
    List<ConnectKeyValue> taskConfig1 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKeyValue1);
        }
    };
    List<ConnectKeyValue> taskConfig2 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKeyValue2);
            add(connectKeyValue3);
        }
    };
    List<ConnectKeyValue> taskConfig3 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKeyValue4);
            add(connectKeyValue5);
            add(connectKeyValue6);
        }
    };
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<String, List<ConnectKeyValue>>() {

        {
            put("connectorConfig1", taskConfig1);
            put("connectorConfig2", taskConfig2);
            put("connectorConfig3", taskConfig3);
        }
    };
    ConnAndTaskConfigs allocate = defaultAllocateConnAndTaskStrategy.allocate(allWorker, curWorker, connectorConfigs, taskConfigs);
    assertNotNull(allocate);
    Map<String, ConnectKeyValue> connectorConfigs3 = allocate.getConnectorConfigs();
    assertNotNull(connectorConfigs3);
    assertNotNull(connectorConfigs3.get("connectorConfig3"));
    assertNull(connectorConfigs3.get("connectorConfig2"));
    assertNull(connectorConfigs3.get("connectorConfig1"));
    Map<String, List<ConnectKeyValue>> taskConfigs1 = allocate.getTaskConfigs();
    assertNotNull(taskConfigs1);
    assertNull(taskConfigs1.get("connectorConfig1"));
    List<ConnectKeyValue> connectorConfig1 = taskConfigs1.get("connectorConfig2");
    assertNotNull(connectorConfig1);
    assertEquals(1, connectorConfig1.size());
    assertEquals(connectKeyValue3, connectorConfig1.get(0));
    List<ConnectKeyValue> connectorConfig2 = taskConfigs1.get("connectorConfig3");
    assertNotNull(connectorConfig2);
    assertEquals(1, connectorConfig2.size());
    assertEquals(connectKeyValue6, connectorConfig2.get(0));
}

Example 5 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class TransferUtilsTest method testToJsonStringToConnAndTaskConfigs.

@Test
public void testToJsonStringToConnAndTaskConfigs() {
    String connectName = "testConnector";
    ConnectKeyValue connectKeyValue = new ConnectKeyValue();
    connectKeyValue.put(RuntimeConfigDefine.CONNECTOR_CLASS, "io.openmessaging.connect.runtime.service.TestConnector");
    connectKeyValue.put(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER, "source-record-converter");
    List<ConnectKeyValue> connectKeyValues = new ArrayList<ConnectKeyValue>(8) {

        {
            add(connectKeyValue);
        }
    };
    Map<String, ConnectKeyValue> connectorConfigs = new HashMap<String, ConnectKeyValue>() {

        {
            put(connectName, connectKeyValue);
        }
    };
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<String, List<ConnectKeyValue>>() {

        {
            put(connectName, connectKeyValues);
        }
    };
    ConnAndTaskConfigs connAndTaskConfigs = new ConnAndTaskConfigs();
    connAndTaskConfigs.setConnectorConfigs(connectorConfigs);
    connAndTaskConfigs.setTaskConfigs(taskConfigs);
    Map<String, String> connectorMap = new HashMap<>();
    Map<String, String> taskMap = new HashMap<>();
    for (String key : connAndTaskConfigs.getConnectorConfigs().keySet()) {
        connectorMap.put(key, TransferUtils.keyValueToString(connAndTaskConfigs.getConnectorConfigs().get(key)));
    }
    for (String key : connAndTaskConfigs.getTaskConfigs().keySet()) {
        taskMap.put(key, TransferUtils.keyValueListToString(connAndTaskConfigs.getTaskConfigs().get(key)));
    }
    String s = TransferUtils.toJsonString(connectorMap, taskMap);
    ConnAndTaskConfigs connAndTaskConfigs1 = TransferUtils.toConnAndTaskConfigs(s);
    Map<String, ConnectKeyValue> connectorConfigs1 = connAndTaskConfigs1.getConnectorConfigs();
    assertNotNull(connAndTaskConfigs1);
    ConnectKeyValue connectKeyValue1 = connectorConfigs1.get(connectName);
    assertNotNull(connectKeyValue1);
    assertEquals("io.openmessaging.connect.runtime.service.TestConnector", connectKeyValue1.getString(RuntimeConfigDefine.CONNECTOR_CLASS));
    assertEquals("source-record-converter", connectKeyValue1.getString(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER));
    Map<String, List<ConnectKeyValue>> taskConfigs1 = connAndTaskConfigs1.getTaskConfigs();
    List<ConnectKeyValue> connectKeyValues1 = taskConfigs1.get(connectName);
    assertNotNull(connectKeyValues1);
    ConnectKeyValue connectKeyValue2 = connectKeyValues1.get(0);
    assertNotNull(connectKeyValue2);
    assertEquals("io.openmessaging.connect.runtime.service.TestConnector", connectKeyValue2.getString(RuntimeConfigDefine.CONNECTOR_CLASS));
    assertEquals("source-record-converter", connectKeyValue2.getString(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER));
}