Examples with ConnectKeyValue org.apache.rocketmq.connect.runtime.common.ConnectKeyValue used on opensource projects

Example 16 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class WorkerConnectorTest method init.

@Before
public void init() {
    ConnectKeyValue connectKeyValue = new ConnectKeyValue();
    connectKeyValue.put("key1", "value1");
    workerConnector = new WorkerConnector("TEST-CONN", new TestConnector(), connectKeyValue, connectorContext);
    workerConnector.start();
}

Example 17 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class TransferUtilsTest method testKeyValue2StringKeyValue.

@Test
public void testKeyValue2StringKeyValue() {
    ConnectKeyValue connectKeyValue = new ConnectKeyValue();
    connectKeyValue.put("key1", 1);
    connectKeyValue.put("key2", 2L);
    connectKeyValue.put("key3", 3.0);
    connectKeyValue.put("key4", "4");
    String s = TransferUtils.keyValueToString(connectKeyValue);
    ConnectKeyValue connectKeyValue1 = TransferUtils.stringToKeyValue(s);
    assertEquals(1, connectKeyValue1.getInt("key1"));
    assertEquals(2L, connectKeyValue1.getLong("key2"));
    assertTrue(Objects.equals(3.0, connectKeyValue1.getDouble("key3")));
    assertEquals("4", connectKeyValue1.getString("key4"));
}

Example 18 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class Worker method startConnectors.

/**
 * Start a collection of connectors with the given configs. If a connector is already started with the same configs,
 * it will not start again. If a connector is already started but not contained in the new configs, it will stop.
 *
 * @param connectorConfigs
 * @param connectController
 * @throws Exception
 */
public synchronized void startConnectors(Map<String, ConnectKeyValue> connectorConfigs, ConnectController connectController) throws Exception {
    Set<WorkerConnector> stoppedConnector = new HashSet<>();
    for (WorkerConnector workerConnector : workingConnectors) {
        String connectorName = workerConnector.getConnectorName();
        ConnectKeyValue keyValue = connectorConfigs.get(connectorName);
        if (null == keyValue || 0 != keyValue.getInt(RuntimeConfigDefine.CONFIG_DELETED)) {
            workerConnector.stop();
            log.info("Connector {} stop", workerConnector.getConnectorName());
            stoppedConnector.add(workerConnector);
        } else if (!keyValue.equals(workerConnector.getKeyValue())) {
            workerConnector.reconfigure(keyValue);
        }
    }
    workingConnectors.removeAll(stoppedConnector);
    if (null == connectorConfigs || 0 == connectorConfigs.size()) {
        return;
    }
    Map<String, ConnectKeyValue> newConnectors = new HashMap<>();
    for (String connectorName : connectorConfigs.keySet()) {
        boolean isNewConnector = true;
        for (WorkerConnector workerConnector : workingConnectors) {
            if (workerConnector.getConnectorName().equals(connectorName)) {
                isNewConnector = false;
                break;
            }
        }
        if (isNewConnector) {
            newConnectors.put(connectorName, connectorConfigs.get(connectorName));
        }
    }
    for (String connectorName : newConnectors.keySet()) {
        ConnectKeyValue keyValue = newConnectors.get(connectorName);
        String connectorClass = keyValue.getString(RuntimeConfigDefine.CONNECTOR_CLASS);
        ClassLoader loader = plugin.getPluginClassLoader(connectorClass);
        final ClassLoader currentThreadLoader = plugin.currentThreadLoader();
        Class clazz;
        boolean isolationFlag = false;
        if (loader instanceof PluginClassLoader) {
            clazz = ((PluginClassLoader) loader).loadClass(connectorClass, false);
            isolationFlag = true;
        } else {
            clazz = Class.forName(connectorClass);
        }
        final Connector connector = (Connector) clazz.getDeclaredConstructor().newInstance();
        WorkerConnector workerConnector = new WorkerConnector(connectorName, connector, connectorConfigs.get(connectorName), new DefaultConnectorContext(connectorName, connectController));
        if (isolationFlag) {
            Plugin.compareAndSwapLoaders(loader);
        }
        workerConnector.initialize();
        workerConnector.start();
        log.info("Connector {} start", workerConnector.getConnectorName());
        Plugin.compareAndSwapLoaders(currentThreadLoader);
        this.workingConnectors.add(workerConnector);
    }
}

Example 19 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class Worker method maintainTaskState.

public void maintainTaskState() throws Exception {
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<>();
    synchronized (latestTaskConfigs) {
        taskConfigs.putAll(latestTaskConfigs);
    }
    boolean needCommitPosition = false;
    // STEP 1: check running tasks and put to error status
    for (Runnable runnable : runningTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        String connectorName = workerTask.getConnectorName();
        ConnectKeyValue taskConfig = workerTask.getTaskConfig();
        List<ConnectKeyValue> keyValues = taskConfigs.get(connectorName);
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.ERROR == state) {
            errorTasks.add(runnable);
            runningTasks.remove(runnable);
        } else if (WorkerTaskState.RUNNING == state) {
            boolean needStop = true;
            if (null != keyValues && keyValues.size() > 0) {
                for (ConnectKeyValue keyValue : keyValues) {
                    if (keyValue.equals(taskConfig)) {
                        needStop = false;
                        break;
                    }
                }
            }
            if (needStop) {
                workerTask.stop();
                log.info("Task stopping, connector name {}, config {}", workerTask.getConnectorName(), workerTask.getTaskConfig());
                runningTasks.remove(runnable);
                stoppingTasks.put(runnable, System.currentTimeMillis());
                needCommitPosition = true;
            }
        } else {
            log.error("[BUG] Illegal State in when checking running tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // If some tasks are closed, synchronize the position.
    if (needCommitPosition) {
        taskPositionCommitService.commitTaskPosition();
    }
    // get new Tasks
    Map<String, List<ConnectKeyValue>> newTasks = new HashMap<>();
    for (String connectorName : taskConfigs.keySet()) {
        for (ConnectKeyValue keyValue : taskConfigs.get(connectorName)) {
            boolean isNewTask = true;
            if (isConfigInSet(keyValue, runningTasks) || isConfigInSet(keyValue, pendingTasks.keySet()) || isConfigInSet(keyValue, errorTasks)) {
                isNewTask = false;
            }
            if (isNewTask) {
                if (!newTasks.containsKey(connectorName)) {
                    newTasks.put(connectorName, new ArrayList<>());
                }
                log.info("Add new tasks,connector name {}, config {}", connectorName, keyValue);
                newTasks.get(connectorName).add(keyValue);
            }
        }
    }
    // STEP 2: try to create new tasks
    for (String connectorName : newTasks.keySet()) {
        for (ConnectKeyValue keyValue : newTasks.get(connectorName)) {
            String taskType = keyValue.getString(RuntimeConfigDefine.TASK_TYPE);
            if (TaskType.DIRECT.name().equalsIgnoreCase(taskType)) {
                createDirectTask(connectorName, keyValue);
                continue;
            }
            String taskClass = keyValue.getString(RuntimeConfigDefine.TASK_CLASS);
            ClassLoader loader = plugin.getPluginClassLoader(taskClass);
            final ClassLoader currentThreadLoader = plugin.currentThreadLoader();
            Class taskClazz;
            boolean isolationFlag = false;
            if (loader instanceof PluginClassLoader) {
                taskClazz = ((PluginClassLoader) loader).loadClass(taskClass, false);
                isolationFlag = true;
            } else {
                taskClazz = Class.forName(taskClass);
            }
            final Task task = (Task) taskClazz.getDeclaredConstructor().newInstance();
            final String converterClazzName = keyValue.getString(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER);
            Converter recordConverter = null;
            if (StringUtils.isNotEmpty(converterClazzName)) {
                Class converterClazz = Class.forName(converterClazzName);
                recordConverter = (Converter) converterClazz.newInstance();
            }
            if (isolationFlag) {
                Plugin.compareAndSwapLoaders(loader);
            }
            if (task instanceof SourceTask) {
                DefaultMQProducer producer = ConnectUtil.initDefaultMQProducer(connectConfig);
                WorkerSourceTask workerSourceTask = new WorkerSourceTask(connectorName, (SourceTask) task, keyValue, positionManagementService, recordConverter, producer, workerState);
                Plugin.compareAndSwapLoaders(currentThreadLoader);
                Future future = taskExecutor.submit(workerSourceTask);
                taskToFutureMap.put(workerSourceTask, future);
                this.pendingTasks.put(workerSourceTask, System.currentTimeMillis());
            } else if (task instanceof SinkTask) {
                DefaultMQPullConsumer consumer = ConnectUtil.initDefaultMQPullConsumer(connectConfig);
                if (connectConfig.isAutoCreateGroupEnable()) {
                    ConnectUtil.createSubGroup(connectConfig, consumer.getConsumerGroup());
                }
                WorkerSinkTask workerSinkTask = new WorkerSinkTask(connectorName, (SinkTask) task, keyValue, offsetManagementService, recordConverter, consumer, workerState);
                Plugin.compareAndSwapLoaders(currentThreadLoader);
                Future future = taskExecutor.submit(workerSinkTask);
                taskToFutureMap.put(workerSinkTask, future);
                this.pendingTasks.put(workerSinkTask, System.currentTimeMillis());
            }
        }
    }
    // STEP 3: check all pending state
    for (Map.Entry<Runnable, Long> entry : pendingTasks.entrySet()) {
        Runnable runnable = entry.getKey();
        Long startTimestamp = entry.getValue();
        Long currentTimeMillis = System.currentTimeMillis();
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.ERROR == state) {
            errorTasks.add(runnable);
            pendingTasks.remove(runnable);
        } else if (WorkerTaskState.RUNNING == state) {
            runningTasks.add(runnable);
            pendingTasks.remove(runnable);
        } else if (WorkerTaskState.NEW == state) {
            log.info("[RACE CONDITION] we checked the pending tasks before state turns to PENDING");
        } else if (WorkerTaskState.PENDING == state) {
            if (currentTimeMillis - startTimestamp > MAX_START_TIMEOUT_MILLS) {
                ((WorkerTask) runnable).timeout();
                pendingTasks.remove(runnable);
                errorTasks.add(runnable);
            }
        } else {
            log.error("[BUG] Illegal State in when checking pending tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // STEP 4 check stopping tasks
    for (Map.Entry<Runnable, Long> entry : stoppingTasks.entrySet()) {
        Runnable runnable = entry.getKey();
        Long stopTimestamp = entry.getValue();
        Long currentTimeMillis = System.currentTimeMillis();
        Future future = taskToFutureMap.get(runnable);
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.STOPPED == state) {
            if (null == future || !future.isDone()) {
                log.error("[BUG] future is null or Stopped task should have its Future.isDone() true, but false");
            }
            stoppingTasks.remove(runnable);
            stoppedTasks.add(runnable);
        } else if (WorkerTaskState.ERROR == state) {
            stoppingTasks.remove(runnable);
            errorTasks.add(runnable);
        } else if (WorkerTaskState.STOPPING == state) {
            if (currentTimeMillis - stopTimestamp > MAX_STOP_TIMEOUT_MILLS) {
                ((WorkerTask) runnable).timeout();
                stoppingTasks.remove(runnable);
                errorTasks.add(runnable);
            }
        } else {
            log.error("[BUG] Illegal State in when checking stopping tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // STEP 5 check errorTasks and stopped tasks
    for (Runnable runnable : errorTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        Future future = taskToFutureMap.get(runnable);
        try {
            if (null != future) {
                future.get(1000, TimeUnit.MILLISECONDS);
            } else {
                log.error("[BUG] errorTasks reference not found in taskFutureMap");
            }
        } catch (ExecutionException e) {
            Throwable t = e.getCause();
        } catch (CancellationException | TimeoutException | InterruptedException e) {
        } finally {
            future.cancel(true);
            workerTask.cleanup();
            taskToFutureMap.remove(runnable);
            errorTasks.remove(runnable);
            cleanedErrorTasks.add(runnable);
        }
    }
    // STEP 5 check errorTasks and stopped tasks
    for (Runnable runnable : stoppedTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        workerTask.cleanup();
        Future future = taskToFutureMap.get(runnable);
        try {
            if (null != future) {
                future.get(1000, TimeUnit.MILLISECONDS);
            } else {
                log.error("[BUG] stopped Tasks reference not found in taskFutureMap");
            }
        } catch (ExecutionException e) {
            Throwable t = e.getCause();
            log.info("[BUG] Stopped Tasks should not throw any exception");
            t.printStackTrace();
        } catch (CancellationException e) {
            log.info("[BUG] Stopped Tasks throws PrintStackTrace");
            e.printStackTrace();
        } catch (TimeoutException e) {
            log.info("[BUG] Stopped Tasks should not throw any exception");
            e.printStackTrace();
        } catch (InterruptedException e) {
            log.info("[BUG] Stopped Tasks should not throw any exception");
            e.printStackTrace();
        } finally {
            future.cancel(true);
            taskToFutureMap.remove(runnable);
            stoppedTasks.remove(runnable);
            cleanedStoppedTasks.add(runnable);
        }
    }
}

Example 20 with ConnectKeyValue

use of org.apache.rocketmq.connect.runtime.common.ConnectKeyValue in project rocketmq-externals by apache.

the class AllocateConnAndTaskStrategyByConsistentHashTest method testAllocate.

@Test
public void testAllocate() {
    AllocateConnAndTaskStrategyByConsistentHash strategy = new AllocateConnAndTaskStrategyByConsistentHash();
    List<String> allWorker = new ArrayList<String>() {

        {
            add("workId1");
            add("workId2");
            add("workId3");
        }
    };
    Map<String, ConnectKeyValue> connectorConfigs = new HashMap<String, ConnectKeyValue>() {

        {
            put("connectorConfig1", new ConnectKeyValue());
            put("connectorConfig2", new ConnectKeyValue());
            put("connectorConfig3", new ConnectKeyValue());
        }
    };
    List<ConnectKeyValue> connectKVs = new ArrayList<>();
    for (int i = 0; i < 6; i++) {
        ConnectKeyValue kv = new ConnectKeyValue();
        kv.put("index", i);
        connectKVs.add(kv);
    }
    List<ConnectKeyValue> taskConfig1 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKVs.get(0));
        }
    };
    List<ConnectKeyValue> taskConfig2 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKVs.get(1));
            add(connectKVs.get(2));
        }
    };
    List<ConnectKeyValue> taskConfig3 = new ArrayList<ConnectKeyValue>() {

        {
            add(connectKVs.get(3));
            add(connectKVs.get(4));
            add(connectKVs.get(5));
        }
    };
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<String, List<ConnectKeyValue>>() {

        {
            put("connectorConfig1", taskConfig1);
            put("connectorConfig2", taskConfig2);
            put("connectorConfig3", taskConfig3);
        }
    };
    Set<String> connectorChecks = new HashSet<>();
    Map<String, List<ConnectKeyValue>> taskChecks = new HashMap<>();
    taskConfigs.keySet().stream().forEach(key -> {
        taskChecks.put(key, new ArrayList<>());
    });
    for (String worker : allWorker) {
        ConnAndTaskConfigs allocate = strategy.allocate(allWorker, worker, connectorConfigs, taskConfigs);
        assertNotNull(allocate);
        allocate.getConnectorConfigs().keySet().forEach(key -> {
            assertFalse(connectorChecks.contains(key));
            connectorChecks.add(key);
        });
        allocate.getTaskConfigs().forEach((connectorName, allocatedTasks) -> {
            List<ConnectKeyValue> checkKVs = taskChecks.computeIfAbsent(connectorName, k -> new ArrayList<>());
            allocatedTasks.forEach(task -> {
                assertFalse(checkKVs.contains(task));
                checkKVs.add(task);
            });
        });
    }
    assertEquals(connectorConfigs.size(), connectorChecks.size());
    long cnt = taskChecks.entrySet().stream().flatMap(entry -> entry.getValue().stream()).count();
    assertEquals(6, cnt);
}