Examples with SwapContents org.apache.nifi.controller.repository.SwapContents used on opensource projects

Example 1 with SwapContents

use of org.apache.nifi.controller.repository.SwapContents in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testRoundTripSerializeDeserializeFullSwapFile.

@Test
public void testRoundTripSerializeDeserializeFullSwapFile() throws IOException, InterruptedException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    long size = 0L;
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
        size += i;
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    final SwapSerializer serializer = new SchemaSwapSerializer();
    try (final OutputStream fos = new FileOutputStream(swapFile);
        final OutputStream out = new BufferedOutputStream(fos)) {
        serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, out);
    }
    final SwapContents contents;
    final SwapDeserializer deserializer = new SchemaSwapDeserializer();
    try (final FileInputStream fis = new FileInputStream(swapFile);
        final InputStream bufferedIn = new BufferedInputStream(fis);
        final DataInputStream dis = new DataInputStream(bufferedIn)) {
        contents = deserializer.deserializeFlowFiles(dis, swapLocation, flowFileQueue, resourceClaimManager);
    }
    final SwapSummary swapSummary = contents.getSummary();
    assertEquals(10000, swapSummary.getQueueSize().getObjectCount());
    assertEquals(size, swapSummary.getQueueSize().getByteCount());
    assertEquals(9999, swapSummary.getMaxFlowFileId().intValue());
    assertEquals(10000, contents.getFlowFiles().size());
    int counter = 0;
    for (final FlowFileRecord flowFile : contents.getFlowFiles()) {
        final int i = counter++;
        assertEquals(String.valueOf(i), flowFile.getAttribute("i"));
        assertEquals(i, flowFile.getSize());
    }
}

Example 2 with SwapContents

use of org.apache.nifi.controller.repository.SwapContents in project nifi by apache.

the class TestSimpleSwapSerializerDeserializer method testRoundTripSerializeDeserialize.

@Test
public void testRoundTripSerializeDeserialize() throws IOException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
    }
    final String queueId = "87bb99fe-412c-49f6-a441-d1b0af4e20b4";
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn(queueId);
    final String swapLocation = "target/testRoundTrip-" + queueId + ".swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    try {
        final SimpleSwapSerializer serializer = new SimpleSwapSerializer();
        try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
            serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
        }
        final SimpleSwapDeserializer deserializer = new SimpleSwapDeserializer();
        final SwapContents swappedIn;
        try (final FileInputStream fis = new FileInputStream(swapFile);
            final DataInputStream dis = new DataInputStream(fis)) {
            swappedIn = deserializer.deserializeFlowFiles(dis, swapLocation, flowFileQueue, resourceClaimManager);
        }
        assertEquals(toSwap.size(), swappedIn.getFlowFiles().size());
        for (int i = 0; i < toSwap.size(); i++) {
            final FlowFileRecord pre = toSwap.get(i);
            final FlowFileRecord post = swappedIn.getFlowFiles().get(i);
            assertEquals(pre.getSize(), post.getSize());
            assertEquals(pre.getAttributes(), post.getAttributes());
            assertEquals(pre.getSize(), post.getSize());
            assertEquals(pre.getId(), post.getId());
            assertEquals(pre.getContentClaim(), post.getContentClaim());
            assertEquals(pre.getContentClaimOffset(), post.getContentClaimOffset());
            assertEquals(pre.getEntryDate(), post.getEntryDate());
            assertEquals(pre.getLastQueueDate(), post.getLastQueueDate());
            assertEquals(pre.getLineageStartDate(), post.getLineageStartDate());
            assertEquals(pre.getPenaltyExpirationMillis(), post.getPenaltyExpirationMillis());
        }
    } finally {
        Files.deleteIfExists(swapFile.toPath());
    }
}

Example 3 with SwapContents

use of org.apache.nifi.controller.repository.SwapContents in project nifi by apache.

the class TestFileSystemSwapManager method testBackwardCompatible.

@Test
public void testBackwardCompatible() throws IOException {
    try (final InputStream fis = new FileInputStream(new File("src/test/resources/old-swap-file.swap"));
        final DataInputStream in = new DataInputStream(new BufferedInputStream(fis))) {
        final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
        Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
        final FileSystemSwapManager swapManager = createSwapManager();
        final SwapContents swapContents = swapManager.peek("src/test/resources/old-swap-file.swap", flowFileQueue);
        final List<FlowFileRecord> records = swapContents.getFlowFiles();
        assertEquals(10000, records.size());
        for (final FlowFileRecord record : records) {
            assertEquals(4, record.getAttributes().size());
            assertEquals("value", record.getAttribute("key"));
        }
    }
}

Example 4 with SwapContents

use of org.apache.nifi.controller.repository.SwapContents in project nifi by apache.

the class StandardFlowFileQueue method dropFlowFiles.

@Override
public DropFlowFileStatus dropFlowFiles(final String requestIdentifier, final String requestor) {
    logger.info("Initiating drop of FlowFiles from {} on behalf of {} (request identifier={})", this, requestor, requestIdentifier);
    // purge any old requests from the map just to keep it clean. But if there are very requests, which is usually the case, then don't bother
    if (dropRequestMap.size() > 10) {
        final List<String> toDrop = new ArrayList<>();
        for (final Map.Entry<String, DropFlowFileRequest> entry : dropRequestMap.entrySet()) {
            final DropFlowFileRequest request = entry.getValue();
            final boolean completed = request.getState() == DropFlowFileState.COMPLETE || request.getState() == DropFlowFileState.FAILURE;
            if (completed && System.currentTimeMillis() - request.getLastUpdated() > TimeUnit.MINUTES.toMillis(5L)) {
                toDrop.add(entry.getKey());
            }
        }
        for (final String requestId : toDrop) {
            dropRequestMap.remove(requestId);
        }
    }
    final DropFlowFileRequest dropRequest = new DropFlowFileRequest(requestIdentifier);
    final QueueSize originalSize = getQueueSize();
    dropRequest.setCurrentSize(originalSize);
    dropRequest.setOriginalSize(originalSize);
    if (originalSize.getObjectCount() == 0) {
        dropRequest.setDroppedSize(originalSize);
        dropRequest.setState(DropFlowFileState.COMPLETE);
        dropRequestMap.put(requestIdentifier, dropRequest);
        return dropRequest;
    }
    final Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            writeLock.lock();
            try {
                dropRequest.setState(DropFlowFileState.DROPPING_FLOWFILES);
                logger.debug("For DropFlowFileRequest {}, original size is {}", requestIdentifier, getQueueSize());
                try {
                    final List<FlowFileRecord> activeQueueRecords = new ArrayList<>(activeQueue);
                    QueueSize droppedSize;
                    try {
                        if (dropRequest.getState() == DropFlowFileState.CANCELED) {
                            logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
                            return;
                        }
                        droppedSize = drop(activeQueueRecords, requestor);
                        logger.debug("For DropFlowFileRequest {}, Dropped {} from active queue", requestIdentifier, droppedSize);
                    } catch (final IOException ioe) {
                        logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
                        logger.error("", ioe);
                        dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
                        return;
                    }
                    activeQueue.clear();
                    incrementActiveQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount());
                    dropRequest.setCurrentSize(getQueueSize());
                    dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
                    final QueueSize swapSize = size.get().swapQueueSize();
                    logger.debug("For DropFlowFileRequest {}, Swap Queue has {} elements, Swapped Record Count = {}, Swapped Content Size = {}", requestIdentifier, swapQueue.size(), swapSize.getObjectCount(), swapSize.getByteCount());
                    if (dropRequest.getState() == DropFlowFileState.CANCELED) {
                        logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
                        return;
                    }
                    try {
                        droppedSize = drop(swapQueue, requestor);
                    } catch (final IOException ioe) {
                        logger.error("Failed to drop the FlowFiles from queue {} due to {}", StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
                        logger.error("", ioe);
                        dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + ioe.toString());
                        return;
                    }
                    swapQueue.clear();
                    dropRequest.setCurrentSize(getQueueSize());
                    dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
                    swapMode = false;
                    incrementSwapQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount(), 0);
                    logger.debug("For DropFlowFileRequest {}, dropped {} from Swap Queue", requestIdentifier, droppedSize);
                    final int swapFileCount = swapLocations.size();
                    final Iterator<String> swapLocationItr = swapLocations.iterator();
                    while (swapLocationItr.hasNext()) {
                        final String swapLocation = swapLocationItr.next();
                        SwapContents swapContents = null;
                        try {
                            if (dropRequest.getState() == DropFlowFileState.CANCELED) {
                                logger.info("Cancel requested for DropFlowFileRequest {}", requestIdentifier);
                                return;
                            }
                            swapContents = swapManager.swapIn(swapLocation, StandardFlowFileQueue.this);
                            droppedSize = drop(swapContents.getFlowFiles(), requestor);
                        } catch (final IncompleteSwapFileException isfe) {
                            swapContents = isfe.getPartialContents();
                            final String warnMsg = "Failed to swap in FlowFiles from Swap File " + swapLocation + " because the file was corrupt. " + "Some FlowFiles may not be dropped from the queue until NiFi is restarted.";
                            logger.warn(warnMsg);
                            if (eventReporter != null) {
                                eventReporter.reportEvent(Severity.WARNING, "Drop FlowFiles", warnMsg);
                            }
                        } catch (final IOException ioe) {
                            logger.error("Failed to swap in FlowFiles from Swap File {} in order to drop the FlowFiles for Connection {} due to {}", swapLocation, StandardFlowFileQueue.this.getIdentifier(), ioe.toString());
                            logger.error("", ioe);
                            if (eventReporter != null) {
                                eventReporter.reportEvent(Severity.ERROR, "Drop FlowFiles", "Failed to swap in FlowFiles from Swap File " + swapLocation + ". The FlowFiles contained in this Swap File will not be dropped from the queue");
                            }
                            dropRequest.setState(DropFlowFileState.FAILURE, "Failed to swap in FlowFiles from Swap File " + swapLocation + " due to " + ioe.toString());
                            if (swapContents != null) {
                                // ensure that we don't lose the FlowFiles from our queue.
                                activeQueue.addAll(swapContents.getFlowFiles());
                            }
                            return;
                        }
                        dropRequest.setDroppedSize(dropRequest.getDroppedSize().add(droppedSize));
                        incrementSwapQueueSize(-droppedSize.getObjectCount(), -droppedSize.getByteCount(), -1);
                        dropRequest.setCurrentSize(getQueueSize());
                        swapLocationItr.remove();
                        logger.debug("For DropFlowFileRequest {}, dropped {} for Swap File {}", requestIdentifier, droppedSize, swapLocation);
                    }
                    logger.debug("Dropped FlowFiles from {} Swap Files", swapFileCount);
                    logger.info("Successfully dropped {} FlowFiles ({} bytes) from Connection with ID {} on behalf of {}", dropRequest.getDroppedSize().getObjectCount(), dropRequest.getDroppedSize().getByteCount(), StandardFlowFileQueue.this.getIdentifier(), requestor);
                    dropRequest.setState(DropFlowFileState.COMPLETE);
                } catch (final Exception e) {
                    logger.error("Failed to drop FlowFiles from Connection with ID {} due to {}", StandardFlowFileQueue.this.getIdentifier(), e.toString());
                    logger.error("", e);
                    dropRequest.setState(DropFlowFileState.FAILURE, "Failed to drop FlowFiles due to " + e.toString());
                }
            } finally {
                writeLock.unlock("Drop FlowFiles");
            }
        }
    }, "Drop FlowFiles for Connection " + getIdentifier());
    t.setDaemon(true);
    t.start();
    dropRequestMap.put(requestIdentifier, dropRequest);
    return dropRequest;
}

Example 5 with SwapContents

use of org.apache.nifi.controller.repository.SwapContents in project nifi by apache.

the class FileSystemSwapManager method swapIn.

@Override
public SwapContents swapIn(final String swapLocation, final FlowFileQueue flowFileQueue) throws IOException {
    final File swapFile = new File(swapLocation);
    final SwapContents swapContents = peek(swapLocation, flowFileQueue);
    flowFileRepository.swapFlowFilesIn(swapFile.getAbsolutePath(), swapContents.getFlowFiles(), flowFileQueue);
    if (!swapFile.delete()) {
        warn("Swapped in FlowFiles from file " + swapFile.getAbsolutePath() + " but failed to delete the file; this file should be cleaned up manually");
    }
    return swapContents;
}