Examples with FlowFileRecord org.apache.nifi.controller.repository.FlowFileRecord used on opensource projects

Example 1 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardConnectionDAO method getContent.

@Override
public DownloadableContent getContent(String id, String flowFileUuid, String requestUri) {
    try {
        final NiFiUser user = NiFiUserUtils.getNiFiUser();
        final Connection connection = locateConnection(id);
        final FlowFileQueue queue = connection.getFlowFileQueue();
        final FlowFileRecord flowFile = queue.getFlowFile(flowFileUuid);
        if (flowFile == null) {
            throw new ResourceNotFoundException(String.format("The FlowFile with UUID %s is no longer in the active queue.", flowFileUuid));
        }
        // get the attributes and ensure appropriate access
        final Map<String, String> attributes = flowFile.getAttributes();
        final Authorizable dataAuthorizable = new DataAuthorizable(connection.getSourceAuthorizable());
        dataAuthorizable.authorize(authorizer, RequestAction.READ, user, attributes);
        // get the filename and fall back to the identifier (should never happen)
        String filename = attributes.get(CoreAttributes.FILENAME.key());
        if (filename == null) {
            filename = flowFileUuid;
        }
        // get the mime-type
        final String type = attributes.get(CoreAttributes.MIME_TYPE.key());
        // get the content
        final InputStream content = flowController.getContent(flowFile, user.getIdentity(), requestUri);
        return new DownloadableContent(filename, type, content);
    } catch (final ContentNotFoundException cnfe) {
        throw new ResourceNotFoundException("Unable to find the specified content.");
    } catch (final IOException ioe) {
        logger.error(String.format("Unable to get the content for flowfile (%s) at this time.", flowFileUuid), ioe);
        throw new IllegalStateException("Unable to get the content at this time.");
    }
}

Example 2 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testRoundTripSerializeDeserializeFullSwapFile.

@Test
public void testRoundTripSerializeDeserializeFullSwapFile() throws IOException, InterruptedException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    long size = 0L;
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
        size += i;
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    final SwapSerializer serializer = new SchemaSwapSerializer();
    try (final OutputStream fos = new FileOutputStream(swapFile);
        final OutputStream out = new BufferedOutputStream(fos)) {
        serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, out);
    }
    final SwapContents contents;
    final SwapDeserializer deserializer = new SchemaSwapDeserializer();
    try (final FileInputStream fis = new FileInputStream(swapFile);
        final InputStream bufferedIn = new BufferedInputStream(fis);
        final DataInputStream dis = new DataInputStream(bufferedIn)) {
        contents = deserializer.deserializeFlowFiles(dis, swapLocation, flowFileQueue, resourceClaimManager);
    }
    final SwapSummary swapSummary = contents.getSummary();
    assertEquals(10000, swapSummary.getQueueSize().getObjectCount());
    assertEquals(size, swapSummary.getQueueSize().getByteCount());
    assertEquals(9999, swapSummary.getMaxFlowFileId().intValue());
    assertEquals(10000, contents.getFlowFiles().size());
    int counter = 0;
    for (final FlowFileRecord flowFile : contents.getFlowFiles()) {
        final int i = counter++;
        assertEquals(String.valueOf(i), flowFile.getAttribute("i"));
        assertEquals(i, flowFile.getSize());
    }
}

Example 3 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testWritePerformance.

@Test
@Ignore("For manual testing, in order to ensure that changes do not negatively impact performance")
public void testWritePerformance() throws IOException, InterruptedException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final int iterations = 1000;
    final long start = System.nanoTime();
    final SwapSerializer serializer = new SchemaSwapSerializer();
    for (int i = 0; i < iterations; i++) {
        try (final OutputStream out = new NullOutputStream()) {
            serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, out);
        }
    }
    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.out.println("Wrote " + iterations + " Swap Files in " + millis + " millis");
}

Example 4 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class TestSimpleSwapSerializerDeserializer method testRoundTripSerializeDeserialize.

@Test
public void testRoundTripSerializeDeserialize() throws IOException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
    }
    final String queueId = "87bb99fe-412c-49f6-a441-d1b0af4e20b4";
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn(queueId);
    final String swapLocation = "target/testRoundTrip-" + queueId + ".swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    try {
        final SimpleSwapSerializer serializer = new SimpleSwapSerializer();
        try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
            serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
        }
        final SimpleSwapDeserializer deserializer = new SimpleSwapDeserializer();
        final SwapContents swappedIn;
        try (final FileInputStream fis = new FileInputStream(swapFile);
            final DataInputStream dis = new DataInputStream(fis)) {
            swappedIn = deserializer.deserializeFlowFiles(dis, swapLocation, flowFileQueue, resourceClaimManager);
        }
        assertEquals(toSwap.size(), swappedIn.getFlowFiles().size());
        for (int i = 0; i < toSwap.size(); i++) {
            final FlowFileRecord pre = toSwap.get(i);
            final FlowFileRecord post = swappedIn.getFlowFiles().get(i);
            assertEquals(pre.getSize(), post.getSize());
            assertEquals(pre.getAttributes(), post.getAttributes());
            assertEquals(pre.getSize(), post.getSize());
            assertEquals(pre.getId(), post.getId());
            assertEquals(pre.getContentClaim(), post.getContentClaim());
            assertEquals(pre.getContentClaimOffset(), post.getContentClaimOffset());
            assertEquals(pre.getEntryDate(), post.getEntryDate());
            assertEquals(pre.getLastQueueDate(), post.getLastQueueDate());
            assertEquals(pre.getLineageStartDate(), post.getLineageStartDate());
            assertEquals(pre.getPenaltyExpirationMillis(), post.getPenaltyExpirationMillis());
        }
    } finally {
        Files.deleteIfExists(swapFile.toPath());
    }
}

Example 5 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class TestFileSystemSwapManager method testBackwardCompatible.

@Test
public void testBackwardCompatible() throws IOException {
    try (final InputStream fis = new FileInputStream(new File("src/test/resources/old-swap-file.swap"));
        final DataInputStream in = new DataInputStream(new BufferedInputStream(fis))) {
        final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
        Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
        final FileSystemSwapManager swapManager = createSwapManager();
        final SwapContents swapContents = swapManager.peek("src/test/resources/old-swap-file.swap", flowFileQueue);
        final List<FlowFileRecord> records = swapContents.getFlowFiles();
        assertEquals(10000, records.size());
        for (final FlowFileRecord record : records) {
            assertEquals(4, record.getAttributes().size());
            assertEquals("value", record.getAttribute("key"));
        }
    }
}