Examples with SourceTask io.openmessaging.connector.api.source.SourceTask used on opensource projects

Example 1 with SourceTask

use of io.openmessaging.connector.api.source.SourceTask in project rocketmq-externals by apache.

the class Worker method createDirectTask.

private void createDirectTask(String connectorName, ConnectKeyValue keyValue) throws Exception {
    String sourceTaskClass = keyValue.getString(RuntimeConfigDefine.SOURCE_TASK_CLASS);
    Task sourceTask = getTask(sourceTaskClass);
    String sinkTaskClass = keyValue.getString(RuntimeConfigDefine.SINK_TASK_CLASS);
    Task sinkTask = getTask(sinkTaskClass);
    WorkerDirectTask workerDirectTask = new WorkerDirectTask(connectorName, (SourceTask) sourceTask, (SinkTask) sinkTask, keyValue, positionManagementService, workerState);
    Future future = taskExecutor.submit(workerDirectTask);
    taskToFutureMap.put(workerDirectTask, future);
    this.pendingTasks.put(workerDirectTask, System.currentTimeMillis());
}

Example 2 with SourceTask

use of io.openmessaging.connector.api.source.SourceTask in project rocketmq-externals by apache.

the class Worker method maintainTaskState.

public void maintainTaskState() throws Exception {
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<>();
    synchronized (latestTaskConfigs) {
        taskConfigs.putAll(latestTaskConfigs);
    }
    boolean needCommitPosition = false;
    // STEP 1: check running tasks and put to error status
    for (Runnable runnable : runningTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        String connectorName = workerTask.getConnectorName();
        ConnectKeyValue taskConfig = workerTask.getTaskConfig();
        List<ConnectKeyValue> keyValues = taskConfigs.get(connectorName);
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.ERROR == state) {
            errorTasks.add(runnable);
            runningTasks.remove(runnable);
        } else if (WorkerTaskState.RUNNING == state) {
            boolean needStop = true;
            if (null != keyValues && keyValues.size() > 0) {
                for (ConnectKeyValue keyValue : keyValues) {
                    if (keyValue.equals(taskConfig)) {
                        needStop = false;
                        break;
                    }
                }
            }
            if (needStop) {
                workerTask.stop();
                log.info("Task stopping, connector name {}, config {}", workerTask.getConnectorName(), workerTask.getTaskConfig());
                runningTasks.remove(runnable);
                stoppingTasks.put(runnable, System.currentTimeMillis());
                needCommitPosition = true;
            }
        } else {
            log.error("[BUG] Illegal State in when checking running tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // If some tasks are closed, synchronize the position.
    if (needCommitPosition) {
        taskPositionCommitService.commitTaskPosition();
    }
    // get new Tasks
    Map<String, List<ConnectKeyValue>> newTasks = new HashMap<>();
    for (String connectorName : taskConfigs.keySet()) {
        for (ConnectKeyValue keyValue : taskConfigs.get(connectorName)) {
            boolean isNewTask = true;
            if (isConfigInSet(keyValue, runningTasks) || isConfigInSet(keyValue, pendingTasks.keySet()) || isConfigInSet(keyValue, errorTasks)) {
                isNewTask = false;
            }
            if (isNewTask) {
                if (!newTasks.containsKey(connectorName)) {
                    newTasks.put(connectorName, new ArrayList<>());
                }
                log.info("Add new tasks,connector name {}, config {}", connectorName, keyValue);
                newTasks.get(connectorName).add(keyValue);
            }
        }
    }
    // STEP 2: try to create new tasks
    for (String connectorName : newTasks.keySet()) {
        for (ConnectKeyValue keyValue : newTasks.get(connectorName)) {
            String taskType = keyValue.getString(RuntimeConfigDefine.TASK_TYPE);
            if (TaskType.DIRECT.name().equalsIgnoreCase(taskType)) {
                createDirectTask(connectorName, keyValue);
                continue;
            }
            String taskClass = keyValue.getString(RuntimeConfigDefine.TASK_CLASS);
            ClassLoader loader = plugin.getPluginClassLoader(taskClass);
            final ClassLoader currentThreadLoader = plugin.currentThreadLoader();
            Class taskClazz;
            boolean isolationFlag = false;
            if (loader instanceof PluginClassLoader) {
                taskClazz = ((PluginClassLoader) loader).loadClass(taskClass, false);
                isolationFlag = true;
            } else {
                taskClazz = Class.forName(taskClass);
            }
            final Task task = (Task) taskClazz.getDeclaredConstructor().newInstance();
            final String converterClazzName = keyValue.getString(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER);
            Converter recordConverter = null;
            if (StringUtils.isNotEmpty(converterClazzName)) {
                Class converterClazz = Class.forName(converterClazzName);
                recordConverter = (Converter) converterClazz.newInstance();
            }
            if (isolationFlag) {
                Plugin.compareAndSwapLoaders(loader);
            }
            if (task instanceof SourceTask) {
                DefaultMQProducer producer = ConnectUtil.initDefaultMQProducer(connectConfig);
                WorkerSourceTask workerSourceTask = new WorkerSourceTask(connectorName, (SourceTask) task, keyValue, positionManagementService, recordConverter, producer, workerState);
                Plugin.compareAndSwapLoaders(currentThreadLoader);
                Future future = taskExecutor.submit(workerSourceTask);
                taskToFutureMap.put(workerSourceTask, future);
                this.pendingTasks.put(workerSourceTask, System.currentTimeMillis());
            } else if (task instanceof SinkTask) {
                DefaultMQPullConsumer consumer = ConnectUtil.initDefaultMQPullConsumer(connectConfig);
                if (connectConfig.isAutoCreateGroupEnable()) {
                    ConnectUtil.createSubGroup(connectConfig, consumer.getConsumerGroup());
                }
                WorkerSinkTask workerSinkTask = new WorkerSinkTask(connectorName, (SinkTask) task, keyValue, offsetManagementService, recordConverter, consumer, workerState);
                Plugin.compareAndSwapLoaders(currentThreadLoader);
                Future future = taskExecutor.submit(workerSinkTask);
                taskToFutureMap.put(workerSinkTask, future);
                this.pendingTasks.put(workerSinkTask, System.currentTimeMillis());
            }
        }
    }
    // STEP 3: check all pending state
    for (Map.Entry<Runnable, Long> entry : pendingTasks.entrySet()) {
        Runnable runnable = entry.getKey();
        Long startTimestamp = entry.getValue();
        Long currentTimeMillis = System.currentTimeMillis();
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.ERROR == state) {
            errorTasks.add(runnable);
            pendingTasks.remove(runnable);
        } else if (WorkerTaskState.RUNNING == state) {
            runningTasks.add(runnable);
            pendingTasks.remove(runnable);
        } else if (WorkerTaskState.NEW == state) {
            log.info("[RACE CONDITION] we checked the pending tasks before state turns to PENDING");
        } else if (WorkerTaskState.PENDING == state) {
            if (currentTimeMillis - startTimestamp > MAX_START_TIMEOUT_MILLS) {
                ((WorkerTask) runnable).timeout();
                pendingTasks.remove(runnable);
                errorTasks.add(runnable);
            }
        } else {
            log.error("[BUG] Illegal State in when checking pending tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // STEP 4 check stopping tasks
    for (Map.Entry<Runnable, Long> entry : stoppingTasks.entrySet()) {
        Runnable runnable = entry.getKey();
        Long stopTimestamp = entry.getValue();
        Long currentTimeMillis = System.currentTimeMillis();
        Future future = taskToFutureMap.get(runnable);
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.STOPPED == state) {
            if (null == future || !future.isDone()) {
                log.error("[BUG] future is null or Stopped task should have its Future.isDone() true, but false");
            }
            stoppingTasks.remove(runnable);
            stoppedTasks.add(runnable);
        } else if (WorkerTaskState.ERROR == state) {
            stoppingTasks.remove(runnable);
            errorTasks.add(runnable);
        } else if (WorkerTaskState.STOPPING == state) {
            if (currentTimeMillis - stopTimestamp > MAX_STOP_TIMEOUT_MILLS) {
                ((WorkerTask) runnable).timeout();
                stoppingTasks.remove(runnable);
                errorTasks.add(runnable);
            }
        } else {
            log.error("[BUG] Illegal State in when checking stopping tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // STEP 5 check errorTasks and stopped tasks
    for (Runnable runnable : errorTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        Future future = taskToFutureMap.get(runnable);
        try {
            if (null != future) {
                future.get(1000, TimeUnit.MILLISECONDS);
            } else {
                log.error("[BUG] errorTasks reference not found in taskFutureMap");
            }
        } catch (ExecutionException e) {
            Throwable t = e.getCause();
        } catch (CancellationException | TimeoutException | InterruptedException e) {
        } finally {
            future.cancel(true);
            workerTask.cleanup();
            taskToFutureMap.remove(runnable);
            errorTasks.remove(runnable);
            cleanedErrorTasks.add(runnable);
        }
    }
    // STEP 5 check errorTasks and stopped tasks
    for (Runnable runnable : stoppedTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        workerTask.cleanup();
        Future future = taskToFutureMap.get(runnable);
        try {
            if (null != future) {
                future.get(1000, TimeUnit.MILLISECONDS);
            } else {
                log.error("[BUG] stopped Tasks reference not found in taskFutureMap");
            }
        } catch (ExecutionException e) {
            Throwable t = e.getCause();
            log.info("[BUG] Stopped Tasks should not throw any exception");
            t.printStackTrace();
        } catch (CancellationException e) {
            log.info("[BUG] Stopped Tasks throws PrintStackTrace");
            e.printStackTrace();
        } catch (TimeoutException e) {
            log.info("[BUG] Stopped Tasks should not throw any exception");
            e.printStackTrace();
        } catch (InterruptedException e) {
            log.info("[BUG] Stopped Tasks should not throw any exception");
            e.printStackTrace();
        } finally {
            future.cancel(true);
            taskToFutureMap.remove(runnable);
            stoppedTasks.remove(runnable);
            cleanedStoppedTasks.add(runnable);
        }
    }
}