Search in sources :

Example 21 with NettyServerConfig

use of org.apache.rocketmq.remoting.netty.NettyServerConfig in project rocketmq-externals by apache.

the class RocketMQLocalCluster method startBroker.

private void startBroker() throws Exception {
    System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setNamesrvAddr(nameserverAddr);
    brokerConfig.setBrokerId(MixAll.MASTER_ID);
    NettyServerConfig nettyServerConfig = new NettyServerConfig();
    nettyServerConfig.setListenPort(10911);
    NettyClientConfig nettyClientConfig = new NettyClientConfig();
    MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
    brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, messageStoreConfig);
    boolean initResult = brokerController.initialize();
    if (!initResult) {
        brokerController.shutdown();
        throw new Exception("Broker init failure!");
    }
    brokerController.start();
}
Also used : MessageStoreConfig(org.apache.rocketmq.store.config.MessageStoreConfig) BrokerConfig(org.apache.rocketmq.common.BrokerConfig) NettyClientConfig(org.apache.rocketmq.remoting.netty.NettyClientConfig) BrokerController(org.apache.rocketmq.broker.BrokerController) NettyServerConfig(org.apache.rocketmq.remoting.netty.NettyServerConfig)

Example 22 with NettyServerConfig

use of org.apache.rocketmq.remoting.netty.NettyServerConfig in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class BrokerController method initialize.

public boolean initialize() throws CloneNotSupportedException {
    // 这几个load 加载 topics.json、 consumerOffset.json、 subscriptionGroup.json、 consumerFilter.json文件
    // 分别将各文件的数据存入 TopicConfigManager、 ConsumerOffsetManager、SubscriptionGroupManager  consumerFilterManager对象中
    boolean result = this.topicConfigManager.load();
    result = result && this.consumerOffsetManager.load();
    result = result && this.subscriptionGroupManager.load();
    result = result && this.consumerFilterManager.load();
    if (result) {
        try {
            // 初始化 DefaultMessageStore 对象,该对象是应用层访问存储层的访问类
            this.messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig);
            this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore);
            // load plugin
            MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig);
            this.messageStore = MessageStoreFactory.build(context, this.messageStore);
            this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager));
        } catch (IOException e) {
            result = false;
            e.printStackTrace();
        }
    }
    // 加载数据
    result = result && this.messageStore.load();
    if (result) {
        // 初始化 Netty 服务端 NettyRemotingServer 对象
        this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
        NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();
        fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2);
        // 定期检测客户端连接,清除不活动的连接
        this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
        // 初始化发送消息线程池
        this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor(this.brokerConfig.getSendMessageThreadPoolNums(), this.brokerConfig.getSendMessageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.sendThreadPoolQueue, new ThreadFactoryImpl("SendMessageThread_"));
        // 初始化拉取消息线程池
        this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor(this.brokerConfig.getPullMessageThreadPoolNums(), this.brokerConfig.getPullMessageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.pullThreadPoolQueue, new ThreadFactoryImpl("PullMessageThread_"));
        // 初始化管理 Broker 线程池
        this.adminBrokerExecutor = Executors.newFixedThreadPool(this.brokerConfig.getAdminBrokerThreadPoolNums(), new ThreadFactoryImpl("AdminBrokerThread_"));
        // 初始化客户端管理线程池
        this.clientManageExecutor = new ThreadPoolExecutor(this.brokerConfig.getClientManageThreadPoolNums(), this.brokerConfig.getClientManageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientManagerThreadPoolQueue, new ThreadFactoryImpl("ClientManageThread_"));
        // 初始化消费端管理线程池
        this.consumerManageExecutor = Executors.newFixedThreadPool(this.brokerConfig.getConsumerManageThreadPoolNums(), new ThreadFactoryImpl("ConsumerManageThread_"));
        // 注册事件处理器
        this.registerProcessor();
        // TODO remove in future
        final long initialDelay = UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis();
        final long period = 1000 * 60 * 60 * 24;
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    // Broker上的一些统计数据
                    BrokerController.this.getBrokerStats().record();
                } catch (Throwable e) {
                    log.error("schedule record error.", e);
                }
            }
        }, initialDelay, period, TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    // 消费进度持久化
                    BrokerController.this.consumerOffsetManager.persist();
                } catch (Throwable e) {
                    log.error("schedule persist consumerOffset error.", e);
                }
            }
        }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    // 消费过滤持久化
                    BrokerController.this.consumerFilterManager.persist();
                } catch (Throwable e) {
                    log.error("schedule persist consumer filter error.", e);
                }
            }
        }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    // 保护broker
                    BrokerController.this.protectBroker();
                } catch (Exception e) {
                    log.error("protectBroker error.", e);
                }
            }
        }, 3, 3, TimeUnit.MINUTES);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    // 打印标记信息
                    BrokerController.this.printWaterMark();
                } catch (Exception e) {
                    log.error("printWaterMark error.", e);
                }
            }
        }, 10, 1, TimeUnit.SECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    // 纯粹打印日志  dispatch落后commit多少字节
                    log.info("dispatch behind commit log {} bytes", BrokerController.this.getMessageStore().dispatchBehindBytes());
                } catch (Throwable e) {
                    log.error("schedule dispatchBehindBytes error.", e);
                }
            }
        }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        if (this.brokerConfig.getNamesrvAddr() != null) {
            this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr());
            log.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr());
        } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        BrokerController.this.brokerOuterAPI.fetchNameServerAddr();
                    } catch (Throwable e) {
                        log.error("ScheduledTask fetchNameServerAddr exception", e);
                    }
                }
            }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
        }
        if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
            if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
                this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
                this.updateMasterHAServerAddrPeriodically = false;
            } else {
                this.updateMasterHAServerAddrPeriodically = true;
            }
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        // slave 定期同步
                        BrokerController.this.slaveSynchronize.syncAll();
                    } catch (Throwable e) {
                        log.error("ScheduledTask syncAll slave exception", e);
                    }
                }
            }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        } else {
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        // 打印master与slave的差距
                        BrokerController.this.printMasterAndSlaveDiff();
                    } catch (Throwable e) {
                        log.error("schedule printMasterAndSlaveDiff error.", e);
                    }
                }
            }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        }
    }
    return result;
}
Also used : NettyRemotingServer(org.apache.rocketmq.remoting.netty.NettyRemotingServer) ThreadFactoryImpl(org.apache.rocketmq.common.ThreadFactoryImpl) BrokerFixedThreadPoolExecutor(org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor) CommitLogDispatcherCalcBitMap(org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap) IOException(java.io.IOException) DefaultMessageStore(org.apache.rocketmq.store.DefaultMessageStore) IOException(java.io.IOException) BrokerFixedThreadPoolExecutor(org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) BrokerStats(org.apache.rocketmq.store.stats.BrokerStats) NettyServerConfig(org.apache.rocketmq.remoting.netty.NettyServerConfig) MessageStorePluginContext(org.apache.rocketmq.broker.plugin.MessageStorePluginContext)

Example 23 with NettyServerConfig

use of org.apache.rocketmq.remoting.netty.NettyServerConfig in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class IntegrationTestBase method createAndStartBroker.

public static BrokerController createAndStartBroker(String nsAddr) {
    String baseDir = createBaseDir();
    BrokerConfig brokerConfig = new BrokerConfig();
    NettyServerConfig nettyServerConfig = new NettyServerConfig();
    NettyClientConfig nettyClientConfig = new NettyClientConfig();
    MessageStoreConfig storeConfig = new MessageStoreConfig();
    brokerConfig.setBrokerName(BROKER_NAME_PREFIX + BROKER_INDEX.getAndIncrement());
    brokerConfig.setBrokerIP1("127.0.0.1");
    brokerConfig.setNamesrvAddr(nsAddr);
    brokerConfig.setEnablePropertyFilter(true);
    storeConfig.setStorePathRootDir(baseDir);
    storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog");
    storeConfig.setHaListenPort(8000 + random.nextInt(1000));
    storeConfig.setMapedFileSizeCommitLog(COMMIT_LOG_SIZE);
    storeConfig.setMaxIndexNum(INDEX_NUM);
    storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);
    nettyServerConfig.setListenPort(10000 + random.nextInt(1000));
    BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, storeConfig);
    try {
        Assert.assertTrue(brokerController.initialize());
        logger.info("Broker Start name:{} addr:{}", brokerConfig.getBrokerName(), brokerController.getBrokerAddr());
        brokerController.start();
    } catch (Exception e) {
        logger.info("Broker start failed");
        System.exit(1);
    }
    BROKER_CONTROLLERS.add(brokerController);
    return brokerController;
}
Also used : MessageStoreConfig(org.apache.rocketmq.store.config.MessageStoreConfig) BrokerConfig(org.apache.rocketmq.common.BrokerConfig) NettyClientConfig(org.apache.rocketmq.remoting.netty.NettyClientConfig) BrokerController(org.apache.rocketmq.broker.BrokerController) NettyServerConfig(org.apache.rocketmq.remoting.netty.NettyServerConfig)

Example 24 with NettyServerConfig

use of org.apache.rocketmq.remoting.netty.NettyServerConfig in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class IntegrationTestBase method createAndStartNamesrv.

public static NamesrvController createAndStartNamesrv() {
    String baseDir = createBaseDir();
    NamesrvConfig namesrvConfig = new NamesrvConfig();
    NettyServerConfig nameServerNettyServerConfig = new NettyServerConfig();
    namesrvConfig.setKvConfigPath(baseDir + SEP + "namesrv" + SEP + "kvConfig.json");
    namesrvConfig.setConfigStorePath(baseDir + SEP + "namesrv" + SEP + "namesrv.properties");
    nameServerNettyServerConfig.setListenPort(9000 + random.nextInt(1000));
    NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig);
    try {
        Assert.assertTrue(namesrvController.initialize());
        logger.info("Name Server Start:{}", nameServerNettyServerConfig.getListenPort());
        namesrvController.start();
    } catch (Exception e) {
        logger.info("Name Server start failed");
        System.exit(1);
    }
    NAMESRV_CONTROLLERS.add(namesrvController);
    return namesrvController;
}
Also used : NamesrvConfig(org.apache.rocketmq.common.namesrv.NamesrvConfig) NettyServerConfig(org.apache.rocketmq.remoting.netty.NettyServerConfig) NamesrvController(org.apache.rocketmq.namesrv.NamesrvController)

Example 25 with NettyServerConfig

use of org.apache.rocketmq.remoting.netty.NettyServerConfig in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class BrokerHousekeepingServiceTest method setup.

@BeforeClass
public static void setup() {
    NamesrvController namesrvController = new NamesrvController(new NamesrvConfig(), new NettyServerConfig());
    brokerHousekeepingService = new BrokerHousekeepingService(namesrvController);
}
Also used : NamesrvConfig(org.apache.rocketmq.common.namesrv.NamesrvConfig) NettyServerConfig(org.apache.rocketmq.remoting.netty.NettyServerConfig) NamesrvController(org.apache.rocketmq.namesrv.NamesrvController) BeforeClass(org.junit.BeforeClass)

Aggregations

NettyServerConfig (org.apache.rocketmq.remoting.netty.NettyServerConfig)46 NamesrvConfig (org.apache.rocketmq.common.namesrv.NamesrvConfig)23 NamesrvController (org.apache.rocketmq.namesrv.NamesrvController)20 MQClientException (org.apache.rocketmq.client.exception.MQClientException)17 BrokerConfig (org.apache.rocketmq.common.BrokerConfig)17 NettyClientConfig (org.apache.rocketmq.remoting.netty.NettyClientConfig)17 MessageStoreConfig (org.apache.rocketmq.store.config.MessageStoreConfig)17 BrokerController (org.apache.rocketmq.broker.BrokerController)13 MQBrokerException (org.apache.rocketmq.client.exception.MQBrokerException)10 UnsupportedEncodingException (java.io.UnsupportedEncodingException)8 RemotingException (org.apache.rocketmq.remoting.exception.RemotingException)8 LoggerContext (ch.qos.logback.classic.LoggerContext)6 JoranConfigurator (ch.qos.logback.classic.joran.JoranConfigurator)6 BufferedInputStream (java.io.BufferedInputStream)6 FileInputStream (java.io.FileInputStream)6 IOException (java.io.IOException)6 InputStream (java.io.InputStream)6 Properties (java.util.Properties)6 Options (org.apache.commons.cli.Options)6 PosixParser (org.apache.commons.cli.PosixParser)6