Examples with ProfileMetadataMessageProcessor io.cdap.cdap.metadata.profile.ProfileMetadataMessageProcessor used on opensource projects

Example 1 with ProfileMetadataMessageProcessor

use of io.cdap.cdap.metadata.profile.ProfileMetadataMessageProcessor in project cdap by caskdata.

the class MetadataSubscriberService method processMessages.

@Override
protected void processMessages(StructuredTableContext structuredTableContext, Iterator<ImmutablePair<String, MetadataMessage>> messages) throws IOException, ConflictException {
    Map<MetadataMessage.Type, MetadataMessageProcessor> processors = new HashMap<>();
    // Loop over all fetched messages and process them with corresponding MetadataMessageProcessor
    while (messages.hasNext()) {
        ImmutablePair<String, MetadataMessage> next = messages.next();
        String messageId = next.getFirst();
        MetadataMessage message = next.getSecond();
        MetadataMessageProcessor processor = processors.computeIfAbsent(message.getType(), type -> {
            switch(type) {
                case LINEAGE:
                    return new DataAccessLineageProcessor();
                case FIELD_LINEAGE:
                    return new FieldLineageProcessor();
                case USAGE:
                    return new UsageProcessor();
                case WORKFLOW_TOKEN:
                case WORKFLOW_STATE:
                    return new WorkflowProcessor();
                case METADATA_OPERATION:
                    return new MetadataOperationProcessor(cConf);
                case PROFILE_ASSIGNMENT:
                case PROFILE_UNASSIGNMENT:
                case ENTITY_CREATION:
                case ENTITY_DELETION:
                    return new ProfileMetadataMessageProcessor(metadataStorage, structuredTableContext, metricsCollectionService);
                default:
                    return null;
            }
        });
        // noinspection ConstantConditions
        if (processor == null) {
            LOG.warn("Unsupported metadata message type {}. Message ignored.", message.getType());
            continue;
        }
        try {
            processor.processMessage(message, structuredTableContext);
            conflictCount = 0;
        } catch (ConflictException e) {
            if (messageId.equals(conflictMessageId)) {
                conflictCount++;
                if (conflictCount >= maxRetriesOnConflict) {
                    LOG.warn("Skipping metadata message {} after processing it has caused {} consecutive conflicts: {}", message, conflictCount, e.getMessage());
                    continue;
                }
            } else {
                conflictMessageId = messageId;
                conflictCount = 1;
            }
            throw e;
        }
    }
}