Examples with MetadataMessage io.cdap.cdap.data2.metadata.writer.MetadataMessage used on opensource projects

Example 6 with MetadataMessage

use of io.cdap.cdap.data2.metadata.writer.MetadataMessage in project cdap by cdapio.

the class MessagingWorkflowStateWriter method setWorkflowToken.

@Override
public void setWorkflowToken(ProgramRunId workflowRunId, WorkflowToken token) {
    MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_TOKEN, workflowRunId, GSON.toJsonTree(token));
    StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build();
    try {
        Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
    } catch (Exception e) {
        // Don't log the workflow token, as it can be large and may contain sensitive data
        throw new RuntimeException("Failed to publish workflow token for workflow run " + workflowRunId, e);
    }
}

Example 7 with MetadataMessage

use of io.cdap.cdap.data2.metadata.writer.MetadataMessage in project cdap by caskdata.

the class MessagingWorkflowStateWriter method addWorkflowNodeState.

@Override
public void addWorkflowNodeState(ProgramRunId workflowRunId, WorkflowNodeStateDetail state) {
    MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_STATE, workflowRunId, GSON.toJsonTree(state));
    StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build();
    try {
        Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
    } catch (Exception e) {
        throw new RuntimeException("Failed to publish workflow node state for workflow run " + workflowRunId + "of node " + state.getNodeId() + " with state " + state.getNodeStatus(), e);
    }
}

Example 8 with MetadataMessage

use of io.cdap.cdap.data2.metadata.writer.MetadataMessage in project cdap by caskdata.

the class MessagingUsageWriter method register.

@Override
public void register(ProgramId programId, DatasetId datasetId) {
    MetadataMessage message = new MetadataMessage(MetadataMessage.Type.USAGE, programId, GSON.toJsonTree(new DatasetUsage(datasetId)));
    StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build();
    try {
        Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
    } catch (Exception e) {
        throw new RuntimeException("Failed to publish usage for " + datasetId + " for program " + programId, e);
    }
}

Example 9 with MetadataMessage

use of io.cdap.cdap.data2.metadata.writer.MetadataMessage in project cdap by caskdata.

the class MetadataSubscriberService method processMessages.

@Override
protected void processMessages(StructuredTableContext structuredTableContext, Iterator<ImmutablePair<String, MetadataMessage>> messages) throws IOException, ConflictException {
    Map<MetadataMessage.Type, MetadataMessageProcessor> processors = new HashMap<>();
    // Loop over all fetched messages and process them with corresponding MetadataMessageProcessor
    while (messages.hasNext()) {
        ImmutablePair<String, MetadataMessage> next = messages.next();
        String messageId = next.getFirst();
        MetadataMessage message = next.getSecond();
        MetadataMessageProcessor processor = processors.computeIfAbsent(message.getType(), type -> {
            switch(type) {
                case LINEAGE:
                    return new DataAccessLineageProcessor();
                case FIELD_LINEAGE:
                    return new FieldLineageProcessor();
                case USAGE:
                    return new UsageProcessor();
                case WORKFLOW_TOKEN:
                case WORKFLOW_STATE:
                    return new WorkflowProcessor();
                case METADATA_OPERATION:
                    return new MetadataOperationProcessor(cConf);
                case PROFILE_ASSIGNMENT:
                case PROFILE_UNASSIGNMENT:
                case ENTITY_CREATION:
                case ENTITY_DELETION:
                    return new ProfileMetadataMessageProcessor(metadataStorage, structuredTableContext, metricsCollectionService);
                default:
                    return null;
            }
        });
        // noinspection ConstantConditions
        if (processor == null) {
            LOG.warn("Unsupported metadata message type {}. Message ignored.", message.getType());
            continue;
        }
        try {
            processor.processMessage(message, structuredTableContext);
            conflictCount = 0;
        } catch (ConflictException e) {
            if (messageId.equals(conflictMessageId)) {
                conflictCount++;
                if (conflictCount >= maxRetriesOnConflict) {
                    LOG.warn("Skipping metadata message {} after processing it has caused {} consecutive conflicts: {}", message, conflictCount, e.getMessage());
                    continue;
                }
            } else {
                conflictMessageId = messageId;
                conflictCount = 1;
            }
            throw e;
        }
    }
}

Example 10 with MetadataMessage

use of io.cdap.cdap.data2.metadata.writer.MetadataMessage in project cdap by cdapio.

the class MessagingUsageWriter method doRegisterAll.

private void doRegisterAll(Iterable<? extends EntityId> users, EntityId entityId) throws Exception {
    // Only record usage from program
    StoreRequest request = StoreRequestBuilder.of(topic).addPayloads(StreamSupport.stream(users.spliterator(), false).filter(ProgramId.class::isInstance).map(ProgramId.class::cast).map(id -> new MetadataMessage(MetadataMessage.Type.USAGE, id, GSON.toJsonTree(new DatasetUsage(entityId)))).map(GSON::toJson).map(s -> s.getBytes(StandardCharsets.UTF_8)).iterator()).build();
    Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
}