Examples with ConnectKeyValue org.apache.rocketmq.connect.runtime.common.ConnectKeyValue used on opensource projects

Example 21 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class WorkerTest method testStartTasks.

@Test
public void testStartTasks() throws Exception {
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<>();
    for (int i = 1; i < 4; i++) {
        List<ConnectKeyValue> connectKeyValues = new ArrayList<>();
        ConnectKeyValue connectKeyValue = new ConnectKeyValue();
        connectKeyValue.getProperties().put("key1", "TEST-CONN-" + i + "1");
        connectKeyValue.getProperties().put("key2", "TEST-CONN-" + i + "2");
        connectKeyValue.getProperties().put(RuntimeConfigDefine.TASK_CLASS, TestSourceTask.class.getName());
        connectKeyValue.getProperties().put(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER, TestConverter.class.getName());
        connectKeyValue.getProperties().put(RuntimeConfigDefine.NAMESRV_ADDR, "127.0.0.1:9876");
        connectKeyValue.getProperties().put(RuntimeConfigDefine.RMQ_PRODUCER_GROUP, UUID.randomUUID().toString());
        connectKeyValue.getProperties().put(RuntimeConfigDefine.OPERATION_TIMEOUT, "3000");
        connectKeyValues.add(connectKeyValue);
        taskConfigs.put("TEST-CONN-" + i, connectKeyValues);
    }
    worker.startTasks(taskConfigs);
    Set<Runnable> sourceTasks = worker.getWorkingTasks();
    assertThat(sourceTasks.size()).isEqualTo(3);
    for (Runnable runnable : sourceTasks) {
        WorkerSourceTask workerSourceTask = null;
        WorkerSinkTask workerSinkTask = null;
        if (runnable instanceof WorkerSourceTask) {
            workerSourceTask = (WorkerSourceTask) runnable;
        } else {
            workerSinkTask = (WorkerSinkTask) runnable;
        }
        String connectorName = null != workerSourceTask ? workerSourceTask.getConnectorName() : workerSinkTask.getConnectorName();
        assertThat(connectorName).isIn("TEST-CONN-0", "TEST-CONN-1", "TEST-CONN-2", "TEST-CONN-3");
    }
}

Example 22 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class RestHandlerTest method testRESTful.

@Test
public void testRESTful() throws Exception {
    URIBuilder uriBuilder = new URIBuilder(String.format(CREATE_CONNECTOR_URL, "testConnectorName"));
    uriBuilder.setParameter("config", "{\"connector-class\": \"org.apache.rocketmq.connect.runtime.connectorwrapper.testimpl.TestConnector\",\"mysqlAddr\": \"112.74.179.68\",\"mysqlPort\": \"3306\",\"mysqlUsername\": \"canal\",\"mysqlPassword\": \"canal\",\"source-record-converter\":\"org.apache.rocketmq.connect.runtime.converter.JsonConverter\"}");
    URI uri = uriBuilder.build();
    HttpGet httpGet = new HttpGet(uri);
    HttpResponse httpResponse = httpClient.execute(httpGet);
    assertEquals(200, httpResponse.getStatusLine().getStatusCode());
    assertEquals("success", EntityUtils.toString(httpResponse.getEntity(), "UTF-8"));
    URIBuilder uriBuilder1 = new URIBuilder(String.format(STOP_CONNECTOR_URL, "testConnectorName"));
    URI uri1 = uriBuilder1.build();
    HttpGet httpGet1 = new HttpGet(uri1);
    HttpResponse httpResponse1 = httpClient.execute(httpGet1);
    assertEquals(200, httpResponse1.getStatusLine().getStatusCode());
    assertEquals("success", EntityUtils.toString(httpResponse1.getEntity(), "UTF-8"));
    URIBuilder uriBuilder2 = new URIBuilder(GET_CLUSTER_INFO_URL);
    URI uri2 = uriBuilder2.build();
    HttpGet httpGet2 = new HttpGet(uri2);
    HttpResponse httpResponse2 = httpClient.execute(httpGet2);
    assertEquals(200, httpResponse2.getStatusLine().getStatusCode());
    assertEquals(JSON.toJSONString(aliveWorker), EntityUtils.toString(httpResponse2.getEntity(), "UTF-8"));
    URIBuilder uriBuilder3 = new URIBuilder(GET_CONFIG_INFO_URL);
    URI uri3 = uriBuilder3.build();
    HttpGet httpGet3 = new HttpGet(uri3);
    HttpResponse httpResponse3 = httpClient.execute(httpGet3);
    assertEquals(200, httpResponse3.getStatusLine().getStatusCode());
    Map<String, Map> formatter = new HashMap<>();
    formatter.put("connectorConfigs", connectorConfigs);
    formatter.put("taskConfigs", taskConfigs);
    assertEquals(JSON.toJSONString(formatter), EntityUtils.toString(httpResponse3.getEntity(), "UTF-8"));
    URIBuilder uriBuilder4 = new URIBuilder(GET_ALLOCATED_CONNECTORS_URL);
    URI uri4 = uriBuilder4.build();
    HttpGet httpGet4 = new HttpGet(uri4);
    HttpResponse httpResponse4 = httpClient.execute(httpGet4);
    assertEquals(200, httpResponse4.getStatusLine().getStatusCode());
    Map<String, ConnectKeyValue> connectors = new HashMap<>();
    for (WorkerConnector workerConnector : workerConnectors) {
        connectors.put(workerConnector.getConnectorName(), workerConnector.getKeyValue());
    }
    assertEquals(JSON.toJSONString(connectors), EntityUtils.toString(httpResponse4.getEntity(), "UTF-8"));
}

Example 23 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class ConfigManagementServiceImplTest method testRemoveConnectorConfig.

@Test
public void testRemoveConnectorConfig() throws Exception {
    configManagementService.putConnectorConfig(connectorName, this.connectKeyValue);
    Map<String, ConnectKeyValue> connectorConfigs = configManagementService.getConnectorConfigs();
    ConnectKeyValue connectKeyValue = connectorConfigs.get(connectorName);
    Map<String, List<ConnectKeyValue>> taskConfigs = configManagementService.getTaskConfigs();
    List<ConnectKeyValue> connectKeyValues = taskConfigs.get(connectorName);
    assertNotNull(connectKeyValue);
    assertNotNull(connectKeyValues);
    configManagementService.removeConnectorConfig(connectorName);
    connectorConfigs = configManagementService.getConnectorConfigs();
    connectKeyValue = connectorConfigs.get(connectorName);
    taskConfigs = configManagementService.getTaskConfigs();
    connectKeyValues = taskConfigs.get(connectorName);
    assertNull(connectKeyValue);
    assertNull(connectKeyValues);
}

Example 24 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class AllocateConnAndTaskStrategyByConsistentHash method allocate.

@Override
public ConnAndTaskConfigs allocate(List<String> allWorker, String curWorker, Map<String, ConnectKeyValue> connectorConfigs, Map<String, List<ConnectKeyValue>> taskConfigs) {
    ConnAndTaskConfigs allocateResult = new ConnAndTaskConfigs();
    if (null == allWorker || 0 == allWorker.size()) {
        return allocateResult;
    }
    Collection<ClientNode> cidNodes = allWorker.stream().map(ClientNode::new).collect(Collectors.toList());
    ConsistentHashRouter router = getRouter(cidNodes);
    connectorConfigs.entrySet().stream().filter(task -> curWorker.equals(router.routeNode(task.getKey()).getKey())).forEach(task -> allocateResult.getConnectorConfigs().put(task.getKey(), task.getValue()));
    for (Map.Entry<String, List<ConnectKeyValue>> connector : taskConfigs.entrySet()) {
        connector.getValue().stream().filter(kv -> curWorker.equals(router.routeNode(kv.toString()).getKey())).forEach(allocateResult.getTaskConfigs().computeIfAbsent(connector.getKey(), k -> new ArrayList<>())::add);
    }
    log.debug("allocate result: " + allocateResult);
    return allocateResult;
}

Example 25 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class RestHandler method getAllocatedConnectors.

private void getAllocatedConnectors(Context context) {
    Set<WorkerConnector> workerConnectors = connectController.getWorker().getWorkingConnectors();
    Set<Runnable> workerTasks = connectController.getWorker().getWorkingTasks();
    Map<String, ConnectKeyValue> connectors = new HashMap<>();
    for (WorkerConnector workerConnector : workerConnectors) {
        connectors.put(workerConnector.getConnectorName(), workerConnector.getKeyValue());
    }
    context.result(JSON.toJSONString(connectors));
}