Examples with ConnAndTaskConfigs org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs used on opensource projects

Example 6 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class AllocateConnAndTaskStrategyByConsistentHashTest method testAllocate.

@Test
public void testAllocate() {
    AllocateConnAndTaskStrategyByConsistentHash strategy = new AllocateConnAndTaskStrategyByConsistentHash();
    List<String> allWorker = new ArrayList<String>() {

        {
            add("workId1");
            add("workId2");
            add("workId3");
        }
    };
    Map<String, ConnectKeyValue> connectorConfigs = new HashMap<String, ConnectKeyValue>() {

        {
            put("connectorConfig1", new ConnectKeyValue());
            put("connectorConfig2", new ConnectKeyValue());
            put("connectorConfig3", new ConnectKeyValue());
        }
    };
    List<ConnectKeyValue> connectKVs = new ArrayList<>();
    for (int i = 0; i < 6; i++) {
        ConnectKeyValue kv = new ConnectKeyValue();
        kv.put("index", i);
        connectKVs.add(kv);
    }
    List<ConnectKeyValue> taskConfig1 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKVs.get(0));
        }
    };
    List<ConnectKeyValue> taskConfig2 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKVs.get(1));
            add(connectKVs.get(2));
        }
    };
    List<ConnectKeyValue> taskConfig3 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKVs.get(3));
            add(connectKVs.get(4));
            add(connectKVs.get(5));
        }
    };
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<String, List<ConnectKeyValue>>() {

        {
            put("connectorConfig1", taskConfig1);
            put("connectorConfig2", taskConfig2);
            put("connectorConfig3", taskConfig3);
        }
    };
    Set<String> connectorChecks = new HashSet<>();
    Map<String, List<ConnectKeyValue>> taskChecks = new HashMap<>();
    taskConfigs.keySet().stream().forEach(key -> {
        taskChecks.put(key, new ArrayList<>());
    });
    for (String worker : allWorker) {
        ConnAndTaskConfigs allocate = strategy.allocate(allWorker, worker, connectorConfigs, taskConfigs);
        assertNotNull(allocate);
        allocate.getConnectorConfigs().keySet().forEach(key -> {
            assertFalse(connectorChecks.contains(key));
            connectorChecks.add(key);
        });
        allocate.getTaskConfigs().forEach((connectorName, allocatedTasks) -> {
            List<ConnectKeyValue> checkKVs = taskChecks.computeIfAbsent(connectorName, k -> new ArrayList<>());
            allocatedTasks.forEach(task -> {
                assertFalse(checkKVs.contains(task));
                checkKVs.add(task);
            });
        });
    }
    assertEquals(connectorConfigs.size(), connectorChecks.size());
    long cnt = taskChecks.entrySet().stream().flatMap(entry -> entry.getValue().stream()).count();
    assertEquals(6, cnt);
}

Example 7 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class PositionManagementServiceImplTest method init.

@Before
public void init() throws Exception {
    connectConfig = new ConnectConfig();
    connectConfig.setHttpPort(8081);
    connectConfig.setNamesrvAddr("localhost:9876");
    connectConfig.setStorePathRootDir(System.getProperty("user.home") + File.separator + "testConnectorStore");
    connectConfig.setRmqConsumerGroup("testConsumerGroup");
    doAnswer(new Answer() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Exception {
            final Message message = invocation.getArgument(0);
            byte[] bytes = message.getBody();
            final Field dataSynchronizerField = PositionManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
            dataSynchronizerField.setAccessible(true);
            BrokerBasedLog<String, Map> dataSynchronizer = (BrokerBasedLog<String, Map>) dataSynchronizerField.get(positionManagementService);
            final Method decodeKeyValueMethod = BrokerBasedLog.class.getDeclaredMethod("decodeKeyValue", byte[].class);
            decodeKeyValueMethod.setAccessible(true);
            Map<String, Map> map = (Map<String, Map>) decodeKeyValueMethod.invoke(dataSynchronizer, bytes);
            final Field dataSynchronizerCallbackField = BrokerBasedLog.class.getDeclaredField("dataSynchronizerCallback");
            dataSynchronizerCallbackField.setAccessible(true);
            final DataSynchronizerCallback<String, Map> dataSynchronizerCallback = (DataSynchronizerCallback<String, Map>) dataSynchronizerCallbackField.get(dataSynchronizer);
            for (String key : map.keySet()) {
                dataSynchronizerCallback.onCompletion(null, key, map.get(key));
            }
            return null;
        }
    }).when(producer).send(any(Message.class), any(SendCallback.class));
    positionManagementService = new PositionManagementServiceImpl(connectConfig);
    final Field dataSynchronizerField = PositionManagementServiceImpl.class.getDeclaredField("dataSynchronizer");
    dataSynchronizerField.setAccessible(true);
    final Field producerField = BrokerBasedLog.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    producerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(positionManagementService), producer);
    final Field consumerField = BrokerBasedLog.class.getDeclaredField("consumer");
    consumerField.setAccessible(true);
    consumerField.set((BrokerBasedLog<String, ConnAndTaskConfigs>) dataSynchronizerField.get(positionManagementService), consumer);
    positionManagementService.start();
    Field positionStoreField = PositionManagementServiceImpl.class.getDeclaredField("positionStore");
    positionStoreField.setAccessible(true);
    positionStore = (KeyValueStore<ByteBuffer, ByteBuffer>) positionStoreField.get(positionManagementService);
    Field needSyncPartitionField = PositionManagementServiceImpl.class.getDeclaredField("needSyncPartition");
    needSyncPartitionField.setAccessible(true);
    needSyncPartition = (ConcurrentSet<ByteBuffer>) needSyncPartitionField.get(positionManagementService);
    sourcePartition = ByteBuffer.wrap("127.0.0.13306".getBytes("UTF-8"));
    JSONObject jsonObject = new JSONObject();
    // jsonObject.put(MysqlConstants.BINLOG_FILENAME, "binlogFilename");
    // jsonObject.put(MysqlConstants.NEXT_POSITION, "100");
    sourcePosition = ByteBuffer.wrap(jsonObject.toJSONString().getBytes());
    positions = new HashMap<ByteBuffer, ByteBuffer>() {

        {
            put(sourcePartition, sourcePosition);
        }
    };
}

Example 8 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class AllocateConnAndTaskStrategyByConsistentHash method allocate.

@Override
public ConnAndTaskConfigs allocate(List<String> allWorker, String curWorker, Map<String, ConnectKeyValue> connectorConfigs, Map<String, List<ConnectKeyValue>> taskConfigs) {
    ConnAndTaskConfigs allocateResult = new ConnAndTaskConfigs();
    if (null == allWorker || 0 == allWorker.size()) {
        return allocateResult;
    }
    Collection<ClientNode> cidNodes = allWorker.stream().map(ClientNode::new).collect(Collectors.toList());
    ConsistentHashRouter router = getRouter(cidNodes);
    connectorConfigs.entrySet().stream().filter(task -> curWorker.equals(router.routeNode(task.getKey()).getKey())).forEach(task -> allocateResult.getConnectorConfigs().put(task.getKey(), task.getValue()));
    for (Map.Entry<String, List<ConnectKeyValue>> connector : taskConfigs.entrySet()) {
        connector.getValue().stream().filter(kv -> curWorker.equals(router.routeNode(kv.toString()).getKey())).forEach(allocateResult.getTaskConfigs().computeIfAbsent(connector.getKey(), k -> new ArrayList<>())::add);
    }
    log.debug("allocate result: " + allocateResult);
    return allocateResult;
}

Example 9 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class ConnAndTaskConfigConverter method objectToByte.

@Override
public byte[] objectToByte(ConnAndTaskConfigs object) {
    try {
        ConnAndTaskConfigs configs = object;
        Map<String, String> connectorMap = new HashMap<>();
        Map<String, String> taskMap = new HashMap<>();
        for (String key : configs.getConnectorConfigs().keySet()) {
            connectorMap.put(key, TransferUtils.keyValueToString(configs.getConnectorConfigs().get(key)));
        }
        for (String key : configs.getTaskConfigs().keySet()) {
            taskMap.put(key, TransferUtils.keyValueListToString(configs.getTaskConfigs().get(key)));
        }
        return TransferUtils.toJsonString(connectorMap, taskMap).getBytes("UTF-8");
    } catch (Exception e) {
        log.error("ConnAndTaskConfigConverter#objectToByte failed", e);
    }
    return new byte[0];
}

Example 10 with ConnAndTaskConfigs

use of org.apache.rocketmq.connect.runtime.common.ConnAndTaskConfigs in project rocketmq-externals by apache.

the class ConnAndTaskConfigConverter method byteToObject.

@Override
public ConnAndTaskConfigs byteToObject(byte[] bytes) {
    try {
        String jsonString = new String(bytes, "UTF-8");
        ConnAndTaskConfigs configs = TransferUtils.toConnAndTaskConfigs(jsonString);
        return configs;
    } catch (UnsupportedEncodingException e) {
        log.error("ConnAndTaskConfigConverter#byteToObject failed", e);
    }
    return null;
}