Examples with AtlasNotificationStringMessage org.apache.atlas.model.notification.AtlasNotificationStringMessage used on opensource projects

Example 1 with AtlasNotificationStringMessage

use of org.apache.atlas.model.notification.AtlasNotificationStringMessage in project atlas by apache.

the class AbstractNotification method createNotificationMessages.

/**
 * Get the notification message JSON from the given object.
 *
 * @param message  the message in object form
 *
 * @return the message as a JSON string
 */
public static void createNotificationMessages(Object message, List<String> msgJsonList) {
    AtlasNotificationMessage<?> notificationMsg = new AtlasNotificationMessage<>(CURRENT_MESSAGE_VERSION, message, getHostAddress(), getCurrentUser());
    String msgJson = AtlasType.toV1Json(notificationMsg);
    boolean msgLengthExceedsLimit = (msgJson.length() * MAX_BYTES_PER_CHAR) > MESSAGE_MAX_LENGTH_BYTES;
    if (msgLengthExceedsLimit) {
        // get utf-8 bytes for msgJson and check for length limit again
        byte[] msgBytes = AtlasNotificationBaseMessage.getBytesUtf8(msgJson);
        msgLengthExceedsLimit = msgBytes.length > MESSAGE_MAX_LENGTH_BYTES;
        if (msgLengthExceedsLimit) {
            String msgId = getNextMessageId();
            CompressionKind compressionKind = CompressionKind.NONE;
            if (MESSAGE_COMPRESSION_ENABLED) {
                byte[] encodedBytes = AtlasNotificationBaseMessage.gzipCompressAndEncodeBase64(msgBytes);
                compressionKind = CompressionKind.GZIP;
                LOG.info("Compressed large message: msgID={}, uncompressed={} bytes, compressed={} bytes", msgId, msgBytes.length, encodedBytes.length);
                msgLengthExceedsLimit = encodedBytes.length > MESSAGE_MAX_LENGTH_BYTES;
                if (!msgLengthExceedsLimit) {
                    // no need to split
                    AtlasNotificationStringMessage compressedMsg = new AtlasNotificationStringMessage(encodedBytes, msgId, compressionKind);
                    // msgJson will not have multi-byte characters here, due to use of encodeBase64() above
                    msgJson = AtlasType.toV1Json(compressedMsg);
                    // not used after this point
                    msgBytes = null;
                } else {
                    // encodedBytes will be split
                    // not used after this point
                    msgJson = null;
                    msgBytes = encodedBytes;
                }
            }
            if (msgLengthExceedsLimit) {
                // compressed messages are already base64-encoded
                byte[] encodedBytes = MESSAGE_COMPRESSION_ENABLED ? msgBytes : AtlasNotificationBaseMessage.encodeBase64(msgBytes);
                int splitCount = encodedBytes.length / MESSAGE_MAX_LENGTH_BYTES;
                if ((encodedBytes.length % MESSAGE_MAX_LENGTH_BYTES) != 0) {
                    splitCount++;
                }
                for (int i = 0, offset = 0; i < splitCount; i++) {
                    int length = MESSAGE_MAX_LENGTH_BYTES;
                    if ((offset + length) > encodedBytes.length) {
                        length = encodedBytes.length - offset;
                    }
                    AtlasNotificationStringMessage splitMsg = new AtlasNotificationStringMessage(encodedBytes, offset, length, msgId, compressionKind, i, splitCount);
                    String splitMsgJson = AtlasType.toV1Json(splitMsg);
                    msgJsonList.add(splitMsgJson);
                    offset += length;
                }
                LOG.info("Split large message: msgID={}, splitCount={}, length={} bytes", msgId, splitCount, encodedBytes.length);
            }
        }
    }
    if (!msgLengthExceedsLimit) {
        msgJsonList.add(msgJson);
    }
}

Example 2 with AtlasNotificationStringMessage

use of org.apache.atlas.model.notification.AtlasNotificationStringMessage in project atlas by apache.

the class AtlasNotificationMessageDeserializer method deserialize.

// ----- MessageDeserializer ---------------------------------------------
@Override
public T deserialize(String messageJson) {
    final T ret;
    messageCountTotal.incrementAndGet();
    messageCountSinceLastInterval.incrementAndGet();
    AtlasNotificationBaseMessage msg = AtlasType.fromV1Json(messageJson, AtlasNotificationBaseMessage.class);
    if (msg == null || msg.getVersion() == null) {
        // older style messages not wrapped with AtlasNotificationMessage
        ret = AtlasType.fromV1Json(messageJson, messageType);
    } else {
        String msgJson = messageJson;
        if (msg.getMsgSplitCount() > 1) {
            // multi-part message
            AtlasNotificationStringMessage splitMsg = AtlasType.fromV1Json(msgJson, AtlasNotificationStringMessage.class);
            checkVersion(splitMsg, msgJson);
            String msgId = splitMsg.getMsgId();
            if (StringUtils.isEmpty(msgId)) {
                LOG.error("Received multi-part message with no message ID. Ignoring message");
                msg = null;
            } else {
                final int splitIdx = splitMsg.getMsgSplitIdx();
                final int splitCount = splitMsg.getMsgSplitCount();
                final SplitMessageAggregator splitMsgs;
                if (splitIdx == 0) {
                    splitMsgs = new SplitMessageAggregator(splitMsg);
                    splitMsgBuffer.put(splitMsgs.getMsgId(), splitMsgs);
                } else {
                    splitMsgs = splitMsgBuffer.get(msgId);
                }
                if (splitMsgs == null) {
                    LOG.error("Received msgID={}: {} of {}, but first message didn't arrive. Ignoring message", msgId, splitIdx + 1, splitCount);
                    msg = null;
                } else if (splitMsgs.getTotalSplitCount() <= splitIdx) {
                    LOG.error("Received msgID={}: {} of {} - out of bounds. Ignoring message", msgId, splitIdx + 1, splitCount);
                    msg = null;
                } else {
                    LOG.info("Received msgID={}: {} of {}", msgId, splitIdx + 1, splitCount);
                    boolean isReady = splitMsgs.add(splitMsg);
                    if (isReady) {
                        // last message
                        splitMsgBuffer.remove(msgId);
                        boolean isValidMessage = true;
                        StringBuilder sb = new StringBuilder();
                        for (int i = 0; i < splitMsgs.getTotalSplitCount(); i++) {
                            splitMsg = splitMsgs.get(i);
                            if (splitMsg == null) {
                                LOG.warn("MsgID={}: message {} of {} is missing. Ignoring message", msgId, i + 1, splitCount);
                                isValidMessage = false;
                                break;
                            }
                            sb.append(splitMsg.getMessage());
                        }
                        if (isValidMessage) {
                            msgJson = sb.toString();
                            if (CompressionKind.GZIP.equals(splitMsg.getMsgCompressionKind())) {
                                byte[] encodedBytes = AtlasNotificationBaseMessage.getBytesUtf8(msgJson);
                                byte[] bytes = AtlasNotificationBaseMessage.decodeBase64AndGzipUncompress(encodedBytes);
                                msgJson = AtlasNotificationBaseMessage.getStringUtf8(bytes);
                                LOG.info("Received msgID={}: splitCount={}, compressed={} bytes, uncompressed={} bytes", msgId, splitCount, encodedBytes.length, bytes.length);
                            } else {
                                byte[] encodedBytes = AtlasNotificationBaseMessage.getBytesUtf8(msgJson);
                                byte[] bytes = AtlasNotificationBaseMessage.decodeBase64(encodedBytes);
                                msgJson = AtlasNotificationBaseMessage.getStringUtf8(bytes);
                                LOG.info("Received msgID={}: splitCount={}, length={} bytes", msgId, splitCount, bytes.length);
                            }
                            msg = AtlasType.fromV1Json(msgJson, AtlasNotificationBaseMessage.class);
                        } else {
                            msg = null;
                        }
                    } else {
                        // more messages to arrive
                        msg = null;
                    }
                }
            }
        }
        if (msg != null) {
            if (CompressionKind.GZIP.equals(msg.getMsgCompressionKind())) {
                AtlasNotificationStringMessage compressedMsg = AtlasType.fromV1Json(msgJson, AtlasNotificationStringMessage.class);
                byte[] encodedBytes = AtlasNotificationBaseMessage.getBytesUtf8(compressedMsg.getMessage());
                byte[] bytes = AtlasNotificationBaseMessage.decodeBase64AndGzipUncompress(encodedBytes);
                msgJson = AtlasNotificationBaseMessage.getStringUtf8(bytes);
                LOG.info("Received msgID={}: compressed={} bytes, uncompressed={} bytes", compressedMsg.getMsgId(), encodedBytes.length, bytes.length);
            }
            AtlasNotificationMessage<T> atlasNotificationMessage = AtlasType.fromV1Json(msgJson, notificationMessageType);
            checkVersion(atlasNotificationMessage, msgJson);
            ret = atlasNotificationMessage.getMessage();
        } else {
            ret = null;
        }
    }
    long now = System.currentTimeMillis();
    long timeSinceLastPurge = now - splitMessagesLastPurgeTime;
    if (timeSinceLastPurge >= splitMessageBufferPurgeIntervalMs) {
        purgeStaleMessages(splitMsgBuffer, now, splitMessageSegmentsWaitTimeMs);
        LOG.info("Notification processing stats: total={}, sinceLastStatsReport={}", messageCountTotal.get(), messageCountSinceLastInterval.getAndSet(0));
        splitMessagesLastPurgeTime = now;
    }
    return ret;
}