Examples with ClientIdentity com.alibaba.otter.canal.protocol.ClientIdentity used on opensource projects

Example 6 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class PeriodMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperMetaManager);
    if (!zooKeeperMetaManager.isStart()) {
        zooKeeperMetaManager.start();
    }
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return zooKeeperMetaManager.listAllSubscribeInfo(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = zooKeeperMetaManager.getCursor(clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

        public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
            // 读取一下zookeeper信息，初始化一次
            MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
            Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
            for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
                // 添加记录到指定batchId
                batches.addPositionRange(entry.getValue(), entry.getKey());
            }
            return batches;
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(new Runnable() {

        public void run() {
            List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
            for (ClientIdentity clientIdentity : tasks) {
                try {
                    // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
                    zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
                    updateCursorTasks.remove(clientIdentity);
                } catch (Throwable e) {
                    // ignore
                    logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
                }
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}

Example 7 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class FileMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(dataDir);
    if (!dataDir.exists()) {
        try {
            FileUtils.forceMkdir(dataDir);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
    if (!dataDir.canRead() || !dataDir.canWrite()) {
        throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
    }
    dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {

        public File apply(String destination) {
            return getDataFile(destination);
        }
    });
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return loadClientIdentity(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(new Runnable() {

        public void run() {
            List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
            for (ClientIdentity clientIdentity : tasks) {
                MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
                try {
                    // 定时将内存中的最新值刷到file中，多次变更只刷一次
                    if (logger.isInfoEnabled()) {
                        LogPosition cursor = (LogPosition) getCursor(clientIdentity);
                        logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] { clientIdentity.getClientId(), cursor.getPostion().getJournalName(), cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(), cursor.getIdentity().getSourceAddress().toString() });
                    }
                    flushDataToFile(clientIdentity.getDestination());
                    updateCursorTasks.remove(clientIdentity);
                } catch (Throwable e) {
                    // ignore
                    logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
                }
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}

Example 8 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class ZooKeeperMetaManager method listAllSubscribeInfo.

public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getDestinationPath(destination);
    List<String> childs = null;
    try {
        childs = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
    // ignore
    }
    if (CollectionUtils.isEmpty(childs)) {
        return new ArrayList<ClientIdentity>();
    }
    List<Short> clientIds = new ArrayList<Short>();
    for (String child : childs) {
        if (StringUtils.isNumeric(child)) {
            clientIds.add(ZookeeperPathUtils.getClientId(child));
        }
    }
    // 进行一个排序
    Collections.sort(clientIds);
    List<ClientIdentity> clientIdentities = Lists.newArrayList();
    for (Short clientId : clientIds) {
        path = ZookeeperPathUtils.getFilterPath(destination, clientId);
        byte[] bytes = zkClientx.readData(path, true);
        String filter = null;
        if (bytes != null) {
            try {
                filter = new String(bytes, ENCODE);
            } catch (UnsupportedEncodingException e) {
                throw new CanalMetaManagerException(e);
            }
        }
        clientIdentities.add(new ClientIdentity(destination, clientId, filter));
    }
    return clientIdentities;
}

Example 9 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class ClientAuthenticationHandler method messageReceived.

public void messageReceived(final ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
    final Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
    switch(packet.getVersion()) {
        case SUPPORTED_VERSION:
        default:
            final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
            // 如果存在订阅信息
            if (StringUtils.isNotEmpty(clientAuth.getDestination()) && StringUtils.isNotEmpty(clientAuth.getClientId())) {
                ClientIdentity clientIdentity = new ClientIdentity(clientAuth.getDestination(), Short.valueOf(clientAuth.getClientId()), clientAuth.getFilter());
                try {
                    MDC.put("destination", clientIdentity.getDestination());
                    embeddedServer.subscribe(clientIdentity);
                    // 设置状态数据
                    ctx.setAttachment(clientIdentity);
                    // 尝试启动，如果已经启动，忽略
                    if (!embeddedServer.isStart(clientIdentity.getDestination())) {
                        ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
                        if (!runningMonitor.isStart()) {
                            runningMonitor.start();
                        }
                    }
                } finally {
                    MDC.remove("destination");
                }
            }
            NettyUtils.ack(ctx.getChannel(), new ChannelFutureListener() {

                public void operationComplete(ChannelFuture future) throws Exception {
                    logger.info("remove unused channel handlers after authentication is done successfully.");
                    ctx.getPipeline().remove(HandshakeInitializationHandler.class.getName());
                    ctx.getPipeline().remove(ClientAuthenticationHandler.class.getName());
                    int readTimeout = defaultSubscriptorDisconnectIdleTimeout;
                    int writeTimeout = defaultSubscriptorDisconnectIdleTimeout;
                    if (clientAuth.getNetReadTimeout() > 0) {
                        readTimeout = clientAuth.getNetReadTimeout();
                    }
                    if (clientAuth.getNetWriteTimeout() > 0) {
                        writeTimeout = clientAuth.getNetWriteTimeout();
                    }
                    IdleStateHandler idleStateHandler = new IdleStateHandler(NettyUtils.hashedWheelTimer, readTimeout, writeTimeout, 0);
                    ctx.getPipeline().addBefore(SessionHandler.class.getName(), IdleStateHandler.class.getName(), idleStateHandler);
                    IdleStateAwareChannelHandler idleStateAwareChannelHandler = new IdleStateAwareChannelHandler() {

                        public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) throws Exception {
                            logger.warn("channel:{} idle timeout exceeds, close channel to save server resources...", ctx.getChannel());
                            ctx.getChannel().close();
                        }
                    };
                    ctx.getPipeline().addBefore(SessionHandler.class.getName(), IdleStateAwareChannelHandler.class.getName(), idleStateAwareChannelHandler);
                }
            });
            break;
    }
}

Example 10 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class SessionHandler method messageReceived.

public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
    logger.info("message receives in session handler...");
    ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
    Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
    ClientIdentity clientIdentity = null;
    try {
        switch(packet.getType()) {
            case SUBSCRIPTION:
                Sub sub = Sub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
                    clientIdentity = new ClientIdentity(sub.getDestination(), Short.valueOf(sub.getClientId()), sub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    // 尝试启动，如果已经启动，忽略
                    if (!embeddedServer.isStart(clientIdentity.getDestination())) {
                        ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
                        if (!runningMonitor.isStart()) {
                            runningMonitor.start();
                        }
                    }
                    embeddedServer.subscribe(clientIdentity);
                    // 设置状态数据
                    ctx.setAttachment(clientIdentity);
                    NettyUtils.ack(ctx.getChannel(), null);
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case UNSUBSCRIPTION:
                Unsub unsub = Unsub.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
                    clientIdentity = new ClientIdentity(unsub.getDestination(), Short.valueOf(unsub.getClientId()), unsub.getFilter());
                    MDC.put("destination", clientIdentity.getDestination());
                    embeddedServer.unsubscribe(clientIdentity);
                    // 尝试关闭
                    stopCanalInstanceIfNecessary(clientIdentity);
                    NettyUtils.ack(ctx.getChannel(), null);
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case GET:
                Get get = CanalPacket.Get.parseFrom(packet.getBody());
                if (StringUtils.isNotEmpty(get.getDestination()) && StringUtils.isNotEmpty(get.getClientId())) {
                    clientIdentity = new ClientIdentity(get.getDestination(), Short.valueOf(get.getClientId()));
                    MDC.put("destination", clientIdentity.getDestination());
                    Message message = null;
                    // } else {
                    if (get.getTimeout() == -1) {
                        // 是否是初始值
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize());
                    } else {
                        TimeUnit unit = convertTimeUnit(get.getUnit());
                        message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize(), get.getTimeout(), unit);
                    }
                    // }
                    Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
                    packetBuilder.setType(PacketType.MESSAGES);
                    Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
                    messageBuilder.setBatchId(message.getId());
                    if (message.getId() != -1 && !CollectionUtils.isEmpty(message.getEntries())) {
                        for (Entry entry : message.getEntries()) {
                            messageBuilder.addMessages(entry.toByteString());
                        }
                    }
                    packetBuilder.setBody(messageBuilder.build().toByteString());
                    // 输出数据
                    NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case CLIENTACK:
                ClientAck ack = CanalPacket.ClientAck.parseFrom(packet.getBody());
                MDC.put("destination", ack.getDestination());
                if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
                    if (ack.getBatchId() == 0L) {
                        NettyUtils.error(402, MessageFormatter.format("batchId should assign value", ack.toString()).getMessage(), ctx.getChannel(), null);
                    } else if (ack.getBatchId() == -1L) {
                    // -1代表上一次get没有数据，直接忽略之
                    // donothing
                    } else {
                        clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
                        embeddedServer.ack(clientIdentity, ack.getBatchId());
                    }
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            case CLIENTROLLBACK:
                ClientRollback rollback = CanalPacket.ClientRollback.parseFrom(packet.getBody());
                MDC.put("destination", rollback.getDestination());
                if (StringUtils.isNotEmpty(rollback.getDestination()) && StringUtils.isNotEmpty(rollback.getClientId())) {
                    clientIdentity = new ClientIdentity(rollback.getDestination(), Short.valueOf(rollback.getClientId()));
                    if (rollback.getBatchId() == 0L) {
                        // 回滚所有批次
                        embeddedServer.rollback(clientIdentity);
                    } else {
                        // 只回滚单个批次
                        embeddedServer.rollback(clientIdentity, rollback.getBatchId());
                    }
                } else {
                    NettyUtils.error(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage(), ctx.getChannel(), null);
                }
                break;
            default:
                NettyUtils.error(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage(), ctx.getChannel(), null);
                break;
        }
    } catch (Throwable exception) {
        NettyUtils.error(400, MessageFormatter.format("something goes wrong with channel:{}, exception={}", ctx.getChannel(), ExceptionUtils.getStackTrace(exception)).getMessage(), ctx.getChannel(), null);
    } finally {
        MDC.remove("destination");
    }
}