Examples with ProvenanceEventType org.apache.nifi.provenance.ProvenanceEventType used on opensource projects

Example 6 with ProvenanceEventType

use of org.apache.nifi.provenance.ProvenanceEventType in project nifi by apache.

the class FlowController method replayFlowFile.

public ProvenanceEventRecord replayFlowFile(final ProvenanceEventRecord event, final NiFiUser user) throws IOException {
    if (event == null) {
        throw new NullPointerException();
    }
    // Check that the event is a valid type.
    final ProvenanceEventType type = event.getEventType();
    if (type == ProvenanceEventType.JOIN) {
        throw new IllegalArgumentException("Cannot replay events that are created from multiple parents");
    }
    // Make sure event has the Content Claim info
    final Long contentSize = event.getPreviousFileSize();
    final String contentClaimId = event.getPreviousContentClaimIdentifier();
    final String contentClaimSection = event.getPreviousContentClaimSection();
    final String contentClaimContainer = event.getPreviousContentClaimContainer();
    if (contentSize == null || contentClaimId == null || contentClaimSection == null || contentClaimContainer == null) {
        throw new IllegalArgumentException("Cannot replay data from Provenance Event because the event does not contain the required Content Claim");
    }
    // Make sure that the source queue exists
    if (event.getSourceQueueIdentifier() == null) {
        throw new IllegalArgumentException("Cannot replay data from Provenance Event because the event does not specify the Source FlowFile Queue");
    }
    final List<Connection> connections = getGroup(getRootGroupId()).findAllConnections();
    FlowFileQueue queue = null;
    for (final Connection connection : connections) {
        if (event.getSourceQueueIdentifier().equals(connection.getIdentifier())) {
            queue = connection.getFlowFileQueue();
            break;
        }
    }
    if (queue == null) {
        throw new IllegalStateException("Cannot replay data from Provenance Event because the Source FlowFile Queue with ID " + event.getSourceQueueIdentifier() + " no longer exists");
    }
    // Create the ContentClaim. To do so, we first need the appropriate Resource Claim. Because we don't know whether or
    // not the Resource Claim is still active, we first call ResourceClaimManager.getResourceClaim. If this returns
    // null, then we know that the Resource Claim is no longer active and can just create a new one that is not writable.
    // It's critical though that we first call getResourceClaim because otherwise, if the Resource Claim is active and we
    // create a new one that is not writable, we could end up archiving or destroying the Resource Claim while it's still
    // being written to by the Content Repository. This is important only because we are creating a FlowFile with this Resource
    // Claim. If, for instance, we are simply creating the claim to request its content, as in #getContentAvailability, etc.
    // then this is not necessary.
    ResourceClaim resourceClaim = resourceClaimManager.getResourceClaim(event.getPreviousContentClaimContainer(), event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier());
    if (resourceClaim == null) {
        resourceClaim = resourceClaimManager.newResourceClaim(event.getPreviousContentClaimContainer(), event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier(), false, false);
    }
    // Increment Claimant Count, since we will now be referencing the Content Claim
    resourceClaimManager.incrementClaimantCount(resourceClaim);
    final long claimOffset = event.getPreviousContentClaimOffset() == null ? 0L : event.getPreviousContentClaimOffset().longValue();
    final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, claimOffset);
    contentClaim.setLength(event.getPreviousFileSize() == null ? -1L : event.getPreviousFileSize());
    if (!contentRepository.isAccessible(contentClaim)) {
        resourceClaimManager.decrementClaimantCount(resourceClaim);
        throw new IllegalStateException("Cannot replay data from Provenance Event because the data is no longer available in the Content Repository");
    }
    final String parentUUID = event.getFlowFileUuid();
    final String newFlowFileUUID = UUID.randomUUID().toString();
    // We need to create a new FlowFile by populating it with information from the
    // Provenance Event. Particularly of note here is that we are setting the FlowFile's
    // contentClaimOffset to 0. This is done for backward compatibility reasons. ContentClaim
    // used to not have a concept of an offset, and the offset was tied only to the FlowFile. This
    // was later refactored, so that the offset was part of the ContentClaim. If we set the offset
    // in both places, we'll end up skipping over that many bytes twice instead of once (once to get
    // to the beginning of the Content Claim and again to get to the offset within that Content Claim).
    // To avoid this, we just always set the offset in the Content Claim itself and set the
    // FlowFileRecord's contentClaimOffset to 0.
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().addAttributes(event.getPreviousAttributes()).contentClaim(contentClaim).contentClaimOffset(// use 0 because we used the content claim offset in the Content Claim itself
    0L).entryDate(System.currentTimeMillis()).id(flowFileRepository.getNextFlowFileSequence()).lineageStart(event.getLineageStartDate(), 0L).size(contentSize.longValue()).addAttribute("flowfile.replay", "true").addAttribute("flowfile.replay.timestamp", String.valueOf(new Date())).addAttribute(CoreAttributes.UUID.key(), newFlowFileUUID).removeAttributes(CoreAttributes.DISCARD_REASON.key(), CoreAttributes.ALTERNATE_IDENTIFIER.key()).build();
    // Register a Provenance Event to indicate that we replayed the data.
    final ProvenanceEventRecord replayEvent = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.REPLAY).addChildUuid(newFlowFileUUID).addParentUuid(parentUUID).setFlowFileUUID(parentUUID).setAttributes(Collections.emptyMap(), flowFileRecord.getAttributes()).setCurrentContentClaim(event.getContentClaimContainer(), event.getContentClaimSection(), event.getContentClaimIdentifier(), event.getContentClaimOffset(), event.getFileSize()).setDetails("Replay requested by " + user.getIdentity()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setLineageStartDate(event.getLineageStartDate()).setComponentType(event.getComponentType()).setComponentId(event.getComponentId()).build();
    provenanceRepository.registerEvent(replayEvent);
    // Update the FlowFile Repository to indicate that we have added the FlowFile to the flow
    final StandardRepositoryRecord record = new StandardRepositoryRecord(queue);
    record.setWorking(flowFileRecord);
    record.setDestination(queue);
    flowFileRepository.updateRepository(Collections.singleton(record));
    // Enqueue the data
    queue.put(flowFileRecord);
    return replayEvent;
}

Example 7 with ProvenanceEventType

use of org.apache.nifi.provenance.ProvenanceEventType in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForExpandChildren.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForExpandChildren() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    final ProvenanceEventRecord firstEvent = createEvent("4444");
    final Map<String, String> previousAttributes = new HashMap<>();
    previousAttributes.put("uuid", "4444");
    final Map<String, String> updatedAttributes = new HashMap<>();
    updatedAttributes.put("updated", "true");
    final ProvenanceEventRecord fork = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.FORK).setAttributes(previousAttributes, updatedAttributes).addChildFlowFile("1234").setComponentId("component-1").setComponentType("unit test").setEventId(idGenerator.getAndIncrement()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setFlowFileUUID("4444").setLineageStartDate(System.currentTimeMillis()).setCurrentContentClaim("container", "section", "unit-test-id", 0L, 1024L).build();
    index.addEvents(eventStore.addEvent(firstEvent).getStorageLocations());
    index.addEvents(eventStore.addEvent(fork).getStorageLocations());
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final NiFiUser user = createUser();
    final EventAuthorizer allowForkEvents = new EventAuthorizer() {

        @Override
        public boolean isAuthorized(ProvenanceEventRecord event) {
            return event.getEventType() == ProvenanceEventType.FORK;
        }

        @Override
        public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
        }
    };
    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 5) {
        final ComputeLineageSubmission submission = index.submitExpandChildren(1L, user, allowForkEvents);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }
    assertEquals(5, nodes.size());
    assertEquals(1L, nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.FLOWFILE_NODE).count());
    assertEquals(4L, nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).count());
    final Map<ProvenanceEventType, List<LineageNode>> eventMap = nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).collect(Collectors.groupingBy(n -> ((ProvenanceEventLineageNode) n).getEventType()));
    assertEquals(2, eventMap.size());
    assertEquals(1, eventMap.get(ProvenanceEventType.FORK).size());
    assertEquals(3, eventMap.get(ProvenanceEventType.UNKNOWN).size());
}

Example 8 with ProvenanceEventType

use of org.apache.nifi.provenance.ProvenanceEventType in project nifi by apache.

the class LookupTableEventRecord method getEvent.

@SuppressWarnings("unchecked")
public static StandardProvenanceEventRecord getEvent(final Record record, final String storageFilename, final long storageByteOffset, final int maxAttributeLength, final long eventIdStartOffset, final long startTimeOffset, final List<String> componentIds, final List<String> componentTypes, final List<String> queueIds, final List<String> eventTypes) {
    final Map<String, String> previousAttributes = truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.PREVIOUS_ATTRIBUTES), maxAttributeLength);
    final Map<String, String> updatedAttributes = truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.UPDATED_ATTRIBUTES), maxAttributeLength);
    final List<String> childUuids = (List<String>) record.getFieldValue(EventFieldNames.CHILD_UUIDS);
    final List<String> parentUuids = (List<String>) record.getFieldValue(EventFieldNames.PARENT_UUIDS);
    final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder();
    builder.setAlternateIdentifierUri((String) record.getFieldValue(EventFieldNames.ALTERNATE_IDENTIFIER));
    builder.setChildUuids(childUuids);
    builder.setDetails((String) record.getFieldValue(EventFieldNames.EVENT_DETAILS));
    builder.setParentUuids(parentUuids);
    builder.setPreviousAttributes(previousAttributes);
    builder.setRelationship((String) record.getFieldValue(EventFieldNames.RELATIONSHIP));
    builder.setSourceSystemFlowFileIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_SYSTEM_FLOWFILE_IDENTIFIER));
    builder.setTransitUri((String) record.getFieldValue(EventFieldNames.TRANSIT_URI));
    builder.setUpdatedAttributes(updatedAttributes);
    builder.setComponentId(readLookupValue(record.getFieldValue(EventFieldNames.COMPONENT_ID), componentIds));
    builder.setComponentType(readLookupValue(record.getFieldValue(EventFieldNames.COMPONENT_TYPE), componentTypes));
    builder.setSourceQueueIdentifier(readLookupValue(record.getFieldValue(EventFieldNames.SOURCE_QUEUE_IDENTIFIER), queueIds));
    // Determine the event type
    final Integer eventTypeOrdinal = (Integer) record.getFieldValue(EventFieldNames.EVENT_TYPE);
    ProvenanceEventType eventType;
    if (eventTypeOrdinal == null || eventTypeOrdinal > eventTypes.size() || eventTypeOrdinal < 0) {
        eventType = ProvenanceEventType.UNKNOWN;
    } else {
        try {
            eventType = ProvenanceEventType.valueOf(eventTypes.get(eventTypeOrdinal));
        } catch (final Exception e) {
            eventType = ProvenanceEventType.UNKNOWN;
        }
    }
    builder.setEventType(eventType);
    // Determine appropriate UUID for the event
    String uuid = null;
    switch(eventType) {
        case CLONE:
        case FORK:
        case REPLAY:
            if (parentUuids != null && !parentUuids.isEmpty()) {
                uuid = parentUuids.get(0);
            }
            break;
        case JOIN:
            if (childUuids != null && !childUuids.isEmpty()) {
                uuid = childUuids.get(0);
            }
            break;
    }
    if (uuid == null) {
        uuid = updatedAttributes == null ? null : updatedAttributes.get(CoreAttributes.UUID.key());
        if (uuid == null) {
            uuid = previousAttributes == null ? null : previousAttributes.get(CoreAttributes.UUID.key());
        }
    }
    builder.setFlowFileUUID(uuid);
    builder.setEventDuration((Integer) record.getFieldValue(EventFieldNames.EVENT_DURATION));
    builder.setEventTime(addLong((Integer) record.getFieldValue(EventFieldNames.EVENT_TIME), startTimeOffset));
    builder.setFlowFileEntryDate(addLong((Integer) record.getFieldValue(EventFieldNames.FLOWFILE_ENTRY_DATE), startTimeOffset));
    builder.setLineageStartDate(addLong((Integer) record.getFieldValue(EventFieldNames.LINEAGE_START_DATE), startTimeOffset));
    final Integer eventId = (Integer) record.getFieldValue(EventFieldNames.EVENT_IDENTIFIER);
    if (eventId != null) {
        builder.setEventId(eventId.longValue() + eventIdStartOffset);
    }
    builder.setStorageLocation(storageFilename, storageByteOffset);
    final Record previousClaimRecord = (Record) record.getFieldValue(EventFieldNames.PREVIOUS_CONTENT_CLAIM);
    if (previousClaimRecord != null) {
        builder.setPreviousContentClaim((String) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_CONTAINER), (String) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_SECTION), (String) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_IDENTIFIER), (Long) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_OFFSET), (Long) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_SIZE));
    }
    final Object contentClaimObject = record.getFieldValue(EventFieldNames.CONTENT_CLAIM);
    // NO_VALUE type
    builder.setCurrentContentClaim(null, null, null, null, 0L);
    if (contentClaimObject != null) {
        if (contentClaimObject instanceof String) {
            final String contentClaimDescription = (String) contentClaimObject;
            switch(contentClaimDescription) {
                case EventFieldNames.UNCHANGED_VALUE:
                    builder.setCurrentContentClaim((String) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_CONTAINER), (String) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_SECTION), (String) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_IDENTIFIER), (Long) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_OFFSET), (Long) previousClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_SIZE));
                    break;
            }
        } else if (contentClaimObject instanceof Record) {
            final Record currentClaimRecord = (Record) contentClaimObject;
            builder.setCurrentContentClaim((String) currentClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_CONTAINER), (String) currentClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_SECTION), (String) currentClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_IDENTIFIER), (Long) currentClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_OFFSET), (Long) currentClaimRecord.getFieldValue(EventFieldNames.CONTENT_CLAIM_SIZE));
        }
    }
    return builder.build();
}