Examples with ConnectKeyValue org.apache.rocketmq.connect.runtime.common.ConnectKeyValue used on opensource projects

Example 1 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class ConfigManagementServiceImpl method putConnectorConfig.

@Override
public String putConnectorConfig(String connectorName, ConnectKeyValue configs) throws Exception {
    ConnectKeyValue exist = connectorKeyValueStore.get(connectorName);
    if (null != exist) {
        Long updateTimestamp = exist.getLong(RuntimeConfigDefine.UPDATE_TIMESTAMP);
        if (null != updateTimestamp) {
            configs.put(RuntimeConfigDefine.UPDATE_TIMESTAMP, updateTimestamp);
        }
    }
    if (configs.equals(exist)) {
        return "Connector with same config already exist.";
    }
    Long currentTimestamp = System.currentTimeMillis();
    configs.put(RuntimeConfigDefine.UPDATE_TIMESTAMP, currentTimestamp);
    for (String requireConfig : RuntimeConfigDefine.REQUEST_CONFIG) {
        if (!configs.containsKey(requireConfig)) {
            return "Request config key: " + requireConfig;
        }
    }
    String connectorClass = configs.getString(RuntimeConfigDefine.CONNECTOR_CLASS);
    ClassLoader classLoader = plugin.getPluginClassLoader(connectorClass);
    Class clazz;
    if (null != classLoader) {
        clazz = Class.forName(connectorClass, true, classLoader);
    } else {
        clazz = Class.forName(connectorClass);
    }
    final Connector connector = (Connector) clazz.getDeclaredConstructor().newInstance();
    String errorMessage = connector.verifyAndSetConfig(configs);
    if (errorMessage != null && errorMessage.length() > 0) {
        return errorMessage;
    }
    connectorKeyValueStore.put(connectorName, configs);
    recomputeTaskConfigs(connectorName, connector, currentTimestamp);
    return "";
}

Example 2 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class RebalanceImpl method doRebalance.

/**
 * Distribute connectors and tasks according to the {@link RebalanceImpl#allocateConnAndTaskStrategy}.
 */
public void doRebalance() {
    List<String> curAliveWorkers = clusterManagementService.getAllAliveWorkers();
    log.info("Current Alive workers : " + curAliveWorkers.size());
    Map<String, ConnectKeyValue> curConnectorConfigs = configManagementService.getConnectorConfigs();
    log.info("Current ConnectorConfigs : " + curConnectorConfigs);
    Map<String, List<ConnectKeyValue>> curTaskConfigs = configManagementService.getTaskConfigs();
    log.info("Current TaskConfigs : " + curTaskConfigs);
    ConnAndTaskConfigs allocateResult = allocateConnAndTaskStrategy.allocate(curAliveWorkers, clusterManagementService.getCurrentWorker(), curConnectorConfigs, curTaskConfigs);
    log.info("Allocated connector:{}", allocateResult.getConnectorConfigs());
    log.info("Allocated task:{}", allocateResult.getTaskConfigs());
    updateProcessConfigsInRebalance(allocateResult);
}

Example 3 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class DefaultAllocateConnAndTaskStrategy method allocate.

@Override
public ConnAndTaskConfigs allocate(List<String> allWorker, String curWorker, Map<String, ConnectKeyValue> connectorConfigs, Map<String, List<ConnectKeyValue>> taskConfigs) {
    ConnAndTaskConfigs allocateResult = new ConnAndTaskConfigs();
    if (null == allWorker || 0 == allWorker.size()) {
        return allocateResult;
    }
    List<String> sortedWorkers = new ArrayList<>(allWorker);
    Collections.sort(sortedWorkers);
    log.debug("sortedWorkers: {}", sortedWorkers);
    Map<String, ConnectKeyValue> sortedConnectorConfigs = getSortedMap(connectorConfigs);
    log.debug("SortedConnectorConfigs: {}", sortedConnectorConfigs);
    Map<String, List<ConnectKeyValue>> sortedTaskConfigs = getSortedMap(taskConfigs);
    log.debug("SortedTaskConfigs: {}", sortedTaskConfigs);
    int index = 0;
    for (String connectorName : sortedConnectorConfigs.keySet()) {
        String allocatedWorker = sortedWorkers.get(index % sortedWorkers.size());
        index++;
        if (!curWorker.equals(allocatedWorker)) {
            continue;
        }
        allocateResult.getConnectorConfigs().put(connectorName, sortedConnectorConfigs.get(connectorName));
    }
    for (String connectorName : sortedTaskConfigs.keySet()) {
        for (ConnectKeyValue keyValue : sortedTaskConfigs.get(connectorName)) {
            String allocatedWorker = sortedWorkers.get(index % sortedWorkers.size());
            index++;
            if (!curWorker.equals(allocatedWorker)) {
                continue;
            }
            if (null == allocateResult.getTaskConfigs().get(connectorName)) {
                allocateResult.getTaskConfigs().put(connectorName, new ArrayList<>());
            }
            allocateResult.getTaskConfigs().get(connectorName).add(keyValue);
        }
    }
    return allocateResult;
}

Example 4 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class ConfigManagementServiceImplTest method init.

@Before
public void init() throws Exception {
    String consumerGroup = UUID.randomUUID().toString();
    String producerGroup = UUID.randomUUID().toString();
    connectConfig = new ConnectConfig();
    connectConfig.setHttpPort(8081);
    connectConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "testConnectorStore");
    connectConfig.setRmqConsumerGroup("testConsumerGroup");
    connectorName = "testConnectorName";
    connectConfig.setRmqConsumerGroup(consumerGroup);
    connectConfig.setRmqProducerGroup(producerGroup);
    connectConfig.setNamesrvAddr("127.0.0.1:9876");
    connectConfig.setRmqMinConsumeThreadNums(1);
    connectConfig.setRmqMaxConsumeThreadNums(32);
    connectConfig.setRmqMessageConsumeTimeout(3 * 1000);
    connectKeyValue = new ConnectKeyValue();
    connectKeyValue.put(RuntimeConfigDefine.CONNECTOR_CLASS, "org.apache.rocketmq.connect.runtime.connectorwrapper.testimpl.TestConnector");
    connectKeyValue.put(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER, "source-record-converter");
    doAnswer(new Answer() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Exception {
            final Message message = invocation.getArgument(0);
            byte[] bytes = message.getBody();
            final Field dataSynchronizerField = ConfigManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
            dataSynchronizerField.setAccessible(true);
            BrokerBasedLog<String, ConnAndTaskConfigs> dataSynchronizer = (BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService);
            final Method decodeKeyValueMethod = BrokerBasedLog.class.getDeclaredMethod("decodeKeyValue", byte[].class);
            decodeKeyValueMethod.setAccessible(true);
            Map<String, ConnAndTaskConfigs> map = (Map<String, ConnAndTaskConfigs>) decodeKeyValueMethod.invoke(dataSynchronizer, bytes);
            final Field dataSynchronizerCallbackField = BrokerBasedLog.class.getDeclaredField("dataSynchronizerCallback");
            dataSynchronizerCallbackField.setAccessible(true);
            final DataSynchronizerCallback<String, ConnAndTaskConfigs> dataSynchronizerCallback = (DataSynchronizerCallback<String, ConnAndTaskConfigs>) dataSynchronizerCallbackField.get(dataSynchronizer);
            for (String key : map.keySet()) {
                dataSynchronizerCallback.onCompletion(null, key, map.get(key));
            }
            return null;
        }
    }).when(producer).send(any(Message.class), any(SendCallback.class));
    configManagementService = new ConfigManagementServiceImpl(connectConfig, plugin);
    final Field connectorKeyValueStoreField = ConfigManagementServiceImpl.class.getDeclaredField("connectorKeyValueStore");
    connectorKeyValueStoreField.setAccessible(true);
    connectorKeyValueStore = (KeyValueStore<String, ConnectKeyValue>) connectorKeyValueStoreField.get(configManagementService);
    final Field taskKeyValueStoreField = ConfigManagementServiceImpl.class.getDeclaredField("taskKeyValueStore");
    taskKeyValueStoreField.setAccessible(true);
    taskKeyValueStore = (KeyValueStore<String, List<ConnectKeyValue>>) taskKeyValueStoreField.get(configManagementService);
    final Field dataSynchronizerField = ConfigManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
    dataSynchronizerField.setAccessible(true);
    final Field producerField = BrokerBasedLog.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    producerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService), producer);
    final Field consumerField = BrokerBasedLog.class.getDeclaredField("consumer");
    consumerField.setAccessible(true);
    consumerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(configManagementService), consumer);
    configManagementService.start();
}

Example 5 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class ConfigManagementServiceImplTest method testPutConnectorConfig.

@Test
public void testPutConnectorConfig() throws Exception {
    ConnectKeyValue connectKeyValue1 = connectorKeyValueStore.get(connectorName);
    List<ConnectKeyValue> connectKeyValues = taskKeyValueStore.get(connectorName);
    assertNull(connectKeyValue1);
    assertNull(connectKeyValues);
    configManagementService.putConnectorConfig(connectorName, connectKeyValue);
    connectKeyValue1 = connectorKeyValueStore.get(connectorName);
    connectKeyValues = taskKeyValueStore.get(connectorName);
    assertNotNull(connectKeyValue1);
    assertNotNull(connectKeyValues);
}