Examples with DefaultMQProducer org.apache.rocketmq.client.producer.DefaultMQProducer used on opensource projects

Example 71 with DefaultMQProducer

use of org.apache.rocketmq.client.producer.DefaultMQProducer in project rocketmq-externals by apache.

the class Worker method maintainTaskState.

public void maintainTaskState() throws Exception {
    Map<String, List<ConnectKeyValue>> taskConfigs = new HashMap<>();
    synchronized (latestTaskConfigs) {
        taskConfigs.putAll(latestTaskConfigs);
    }
    boolean needCommitPosition = false;
    // STEP 1: check running tasks and put to error status
    for (Runnable runnable : runningTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        String connectorName = workerTask.getConnectorName();
        ConnectKeyValue taskConfig = workerTask.getTaskConfig();
        List<ConnectKeyValue> keyValues = taskConfigs.get(connectorName);
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.ERROR == state) {
            errorTasks.add(runnable);
            runningTasks.remove(runnable);
        } else if (WorkerTaskState.RUNNING == state) {
            boolean needStop = true;
            if (null != keyValues && keyValues.size() > 0) {
                for (ConnectKeyValue keyValue : keyValues) {
                    if (keyValue.equals(taskConfig)) {
                        needStop = false;
                        break;
                    }
                }
            }
            if (needStop) {
                workerTask.stop();
                log.info("Task stopping, connector name {}, config {}", workerTask.getConnectorName(), workerTask.getTaskConfig());
                runningTasks.remove(runnable);
                stoppingTasks.put(runnable, System.currentTimeMillis());
                needCommitPosition = true;
            }
        } else {
            log.error("[BUG] Illegal State in when checking running tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // If some tasks are closed, synchronize the position.
    if (needCommitPosition) {
        taskPositionCommitService.commitTaskPosition();
    }
    // get new Tasks
    Map<String, List<ConnectKeyValue>> newTasks = new HashMap<>();
    for (String connectorName : taskConfigs.keySet()) {
        for (ConnectKeyValue keyValue : taskConfigs.get(connectorName)) {
            boolean isNewTask = true;
            if (isConfigInSet(keyValue, runningTasks) || isConfigInSet(keyValue, pendingTasks.keySet()) || isConfigInSet(keyValue, errorTasks)) {
                isNewTask = false;
            }
            if (isNewTask) {
                if (!newTasks.containsKey(connectorName)) {
                    newTasks.put(connectorName, new ArrayList<>());
                }
                log.info("Add new tasks,connector name {}, config {}", connectorName, keyValue);
                newTasks.get(connectorName).add(keyValue);
            }
        }
    }
    // STEP 2: try to create new tasks
    for (String connectorName : newTasks.keySet()) {
        for (ConnectKeyValue keyValue : newTasks.get(connectorName)) {
            String taskType = keyValue.getString(RuntimeConfigDefine.TASK_TYPE);
            if (TaskType.DIRECT.name().equalsIgnoreCase(taskType)) {
                createDirectTask(connectorName, keyValue);
                continue;
            }
            String taskClass = keyValue.getString(RuntimeConfigDefine.TASK_CLASS);
            ClassLoader loader = plugin.getPluginClassLoader(taskClass);
            final ClassLoader currentThreadLoader = plugin.currentThreadLoader();
            Class taskClazz;
            boolean isolationFlag = false;
            if (loader instanceof PluginClassLoader) {
                taskClazz = ((PluginClassLoader) loader).loadClass(taskClass, false);
                isolationFlag = true;
            } else {
                taskClazz = Class.forName(taskClass);
            }
            final Task task = (Task) taskClazz.getDeclaredConstructor().newInstance();
            final String converterClazzName = keyValue.getString(RuntimeConfigDefine.SOURCE_RECORD_CONVERTER);
            Converter recordConverter = null;
            if (StringUtils.isNotEmpty(converterClazzName)) {
                Class converterClazz = Class.forName(converterClazzName);
                recordConverter = (Converter) converterClazz.newInstance();
            }
            if (isolationFlag) {
                Plugin.compareAndSwapLoaders(loader);
            }
            if (task instanceof SourceTask) {
                DefaultMQProducer producer = ConnectUtil.initDefaultMQProducer(connectConfig);
                WorkerSourceTask workerSourceTask = new WorkerSourceTask(connectorName, (SourceTask) task, keyValue, positionManagementService, recordConverter, producer, workerState);
                Plugin.compareAndSwapLoaders(currentThreadLoader);
                Future future = taskExecutor.submit(workerSourceTask);
                taskToFutureMap.put(workerSourceTask, future);
                this.pendingTasks.put(workerSourceTask, System.currentTimeMillis());
            } else if (task instanceof SinkTask) {
                DefaultMQPullConsumer consumer = ConnectUtil.initDefaultMQPullConsumer(connectConfig);
                if (connectConfig.isAutoCreateGroupEnable()) {
                    ConnectUtil.createSubGroup(connectConfig, consumer.getConsumerGroup());
                }
                WorkerSinkTask workerSinkTask = new WorkerSinkTask(connectorName, (SinkTask) task, keyValue, offsetManagementService, recordConverter, consumer, workerState);
                Plugin.compareAndSwapLoaders(currentThreadLoader);
                Future future = taskExecutor.submit(workerSinkTask);
                taskToFutureMap.put(workerSinkTask, future);
                this.pendingTasks.put(workerSinkTask, System.currentTimeMillis());
            }
        }
    }
    // STEP 3: check all pending state
    for (Map.Entry<Runnable, Long> entry : pendingTasks.entrySet()) {
        Runnable runnable = entry.getKey();
        Long startTimestamp = entry.getValue();
        Long currentTimeMillis = System.currentTimeMillis();
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.ERROR == state) {
            errorTasks.add(runnable);
            pendingTasks.remove(runnable);
        } else if (WorkerTaskState.RUNNING == state) {
            runningTasks.add(runnable);
            pendingTasks.remove(runnable);
        } else if (WorkerTaskState.NEW == state) {
            log.info("[RACE CONDITION] we checked the pending tasks before state turns to PENDING");
        } else if (WorkerTaskState.PENDING == state) {
            if (currentTimeMillis - startTimestamp > MAX_START_TIMEOUT_MILLS) {
                ((WorkerTask) runnable).timeout();
                pendingTasks.remove(runnable);
                errorTasks.add(runnable);
            }
        } else {
            log.error("[BUG] Illegal State in when checking pending tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // STEP 4 check stopping tasks
    for (Map.Entry<Runnable, Long> entry : stoppingTasks.entrySet()) {
        Runnable runnable = entry.getKey();
        Long stopTimestamp = entry.getValue();
        Long currentTimeMillis = System.currentTimeMillis();
        Future future = taskToFutureMap.get(runnable);
        WorkerTaskState state = ((WorkerTask) runnable).getState();
        if (WorkerTaskState.STOPPED == state) {
            if (null == future || !future.isDone()) {
                log.error("[BUG] future is null or Stopped task should have its Future.isDone() true, but false");
            }
            stoppingTasks.remove(runnable);
            stoppedTasks.add(runnable);
        } else if (WorkerTaskState.ERROR == state) {
            stoppingTasks.remove(runnable);
            errorTasks.add(runnable);
        } else if (WorkerTaskState.STOPPING == state) {
            if (currentTimeMillis - stopTimestamp > MAX_STOP_TIMEOUT_MILLS) {
                ((WorkerTask) runnable).timeout();
                stoppingTasks.remove(runnable);
                errorTasks.add(runnable);
            }
        } else {
            log.error("[BUG] Illegal State in when checking stopping tasks, {} is in {} state", ((WorkerTask) runnable).getConnectorName(), state.toString());
        }
    }
    // STEP 5 check errorTasks and stopped tasks
    for (Runnable runnable : errorTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        Future future = taskToFutureMap.get(runnable);
        try {
            if (null != future) {
                future.get(1000, TimeUnit.MILLISECONDS);
            } else {
                log.error("[BUG] errorTasks reference not found in taskFutureMap");
            }
        } catch (ExecutionException e) {
            Throwable t = e.getCause();
        } catch (CancellationException | TimeoutException | InterruptedException e) {
        } finally {
            future.cancel(true);
            workerTask.cleanup();
            taskToFutureMap.remove(runnable);
            errorTasks.remove(runnable);
            cleanedErrorTasks.add(runnable);
        }
    }
    // STEP 5 check errorTasks and stopped tasks
    for (Runnable runnable : stoppedTasks) {
        WorkerTask workerTask = (WorkerTask) runnable;
        workerTask.cleanup();
        Future future = taskToFutureMap.get(runnable);
        try {
            if (null != future) {
                future.get(1000, TimeUnit.MILLISECONDS);
            } else {
                log.error("[BUG] stopped Tasks reference not found in taskFutureMap");
            }
        } catch (ExecutionException e) {
            Throwable t = e.getCause();
            log.info("[BUG] Stopped Tasks should not throw any exception");
            t.printStackTrace();
        } catch (CancellationException e) {
            log.info("[BUG] Stopped Tasks throws PrintStackTrace");
            e.printStackTrace();
        } catch (TimeoutException e) {
            log.info("[BUG] Stopped Tasks should not throw any exception");
            e.printStackTrace();
        } catch (InterruptedException e) {
            log.info("[BUG] Stopped Tasks should not throw any exception");
            e.printStackTrace();
        } finally {
            future.cancel(true);
            taskToFutureMap.remove(runnable);
            stoppedTasks.remove(runnable);
            cleanedStoppedTasks.add(runnable);
        }
    }
}

Example 72 with DefaultMQProducer

use of org.apache.rocketmq.client.producer.DefaultMQProducer in project rocketmq-externals by apache.

the class RocketMQSourceTest method testEvent.

@Test
public void testEvent() throws EventDeliveryException, MQBrokerException, MQClientException, InterruptedException, UnsupportedEncodingException, RemotingException {
    // publish  message before flume-source start
    DefaultMQProducer producer = new DefaultMQProducer(producerGroup);
    producer.setNamesrvAddr(nameServer);
    String sendMsg = "\"Hello Flume\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-dd hh:mm:ss");
    producer.start();
    Message msg = new Message(TOPIC_DEFAULT, tag, sendMsg.getBytes("UTF-8"));
    SendResult sendResult = producer.send(msg);
    // start source
    Context context = new Context();
    context.put(NAME_SERVER_CONFIG, nameServer);
    context.put(TAG_CONFIG, tag);
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    ChannelSelector channelSelector = new ReplicatingChannelSelector();
    channelSelector.setChannels(channels);
    ChannelProcessor channelProcessor = new ChannelProcessor(channelSelector);
    RocketMQSource source = new RocketMQSource();
    source.setChannelProcessor(channelProcessor);
    Configurables.configure(source, context);
    source.start();
    // wait for rebalanceImpl start
    Thread.sleep(2000);
    sendMsg = "\"Hello Flume\"" + "," + DateFormatUtils.format(new Date(), "yyyy-MM-dd hh:mm:ss");
    msg = new Message(TOPIC_DEFAULT, tag, sendMsg.getBytes("UTF-8"));
    sendResult = producer.send(msg);
    log.info("publish message : {}, sendResult:{}", sendMsg, sendResult);
    PollableSource.Status status = source.process();
    if (status == PollableSource.Status.BACKOFF) {
        fail("Error");
    }
    /*
        wait for processQueueTable init
         */
    Thread.sleep(1000);
    producer.shutdown();
    source.stop();
    /*
        mock flume sink
         */
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();
    if (event == null) {
        transaction.commit();
        fail("Error");
    }
    byte[] body = event.getBody();
    String receiveMsg = new String(body, "UTF-8");
    log.info("receive message : {}", receiveMsg);
    assertEquals(sendMsg, receiveMsg);
}

Example 73 with DefaultMQProducer

use of org.apache.rocketmq.client.producer.DefaultMQProducer in project spring-boot-starter-samples by vindell.

the class BroadcastProducer method main.

public static void main(String[] args) throws Exception {
    DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName");
    producer.start();
    for (int i = 0; i < 100; i++) {
        Message msg = new Message("TopicTest", "TagA", "OrderID188", "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET));
        SendResult sendResult = producer.send(msg);
        System.out.printf("%s%n", sendResult);
    }
    producer.shutdown();
}

Example 74 with DefaultMQProducer

use of org.apache.rocketmq.client.producer.DefaultMQProducer in project spring-boot-starter-samples by vindell.

the class OrderedProducer method main.

public static void main(String[] args) throws Exception {
    // Instantiate with a producer group name.
    MQProducer producer = new DefaultMQProducer("example_group_name");
    // Launch the instance.
    producer.start();
    String[] tags = new String[] { "TagA", "TagB", "TagC", "TagD", "TagE" };
    for (int i = 0; i < 100; i++) {
        int orderId = i % 10;
        // Create a message instance, specifying topic, tag and message body.
        Message msg = new Message("TopicTestjjj", tags[i % tags.length], "KEY" + i, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET));
        SendResult sendResult = producer.send(msg, new MessageQueueSelector() {

            @Override
            public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
                Integer id = (Integer) arg;
                int index = id % mqs.size();
                return mqs.get(index);
            }
        }, orderId);
        System.out.printf("%s%n", sendResult);
    }
    // server shutdown
    producer.shutdown();
}

Example 75 with DefaultMQProducer

use of org.apache.rocketmq.client.producer.DefaultMQProducer in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class SqlProducer method main.

public static void main(String[] args) {
    DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name");
    try {
        producer.start();
    } catch (MQClientException e) {
        e.printStackTrace();
        return;
    }
    for (int i = 0; i < 10; i++) {
        try {
            String tag;
            int div = i % 3;
            if (div == 0) {
                tag = "TagA";
            } else if (div == 1) {
                tag = "TagB";
            } else {
                tag = "TagC";
            }
            Message msg = new Message("TopicTest", tag, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET));
            msg.putUserProperty("a", String.valueOf(i));
            SendResult sendResult = producer.send(msg);
            System.out.printf("%s%n", sendResult);
        } catch (Exception e) {
            e.printStackTrace();
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e1) {
                e1.printStackTrace();
            }
        }
    }
    producer.shutdown();
}