Examples with SwapSummary org.apache.nifi.controller.repository.SwapSummary used on opensource projects

Example 1 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testRoundTripSerializeDeserializeFullSwapFile.

@Test
public void testRoundTripSerializeDeserializeFullSwapFile() throws IOException, InterruptedException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    long size = 0L;
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
        size += i;
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    final SwapSerializer serializer = new SchemaSwapSerializer();
    try (final OutputStream fos = new FileOutputStream(swapFile);
        final OutputStream out = new BufferedOutputStream(fos)) {
        serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, out);
    }
    final SwapContents contents;
    final SwapDeserializer deserializer = new SchemaSwapDeserializer();
    try (final FileInputStream fis = new FileInputStream(swapFile);
        final InputStream bufferedIn = new BufferedInputStream(fis);
        final DataInputStream dis = new DataInputStream(bufferedIn)) {
        contents = deserializer.deserializeFlowFiles(dis, swapLocation, flowFileQueue, resourceClaimManager);
    }
    final SwapSummary swapSummary = contents.getSummary();
    assertEquals(10000, swapSummary.getQueueSize().getObjectCount());
    assertEquals(size, swapSummary.getQueueSize().getByteCount());
    assertEquals(9999, swapSummary.getMaxFlowFileId().intValue());
    assertEquals(10000, contents.getFlowFiles().size());
    int counter = 0;
    for (final FlowFileRecord flowFile : contents.getFlowFiles()) {
        final int i = counter++;
        assertEquals(String.valueOf(i), flowFile.getAttribute("i"));
        assertEquals(i, flowFile.getSize());
    }
}

Example 2 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class StandardFlowFileQueue method recoverSwappedFlowFiles.

@Override
public SwapSummary recoverSwappedFlowFiles() {
    int swapFlowFileCount = 0;
    long swapByteCount = 0L;
    Long maxId = null;
    List<ResourceClaim> resourceClaims = new ArrayList<>();
    final long startNanos = System.nanoTime();
    writeLock.lock();
    try {
        final List<String> swapLocations;
        try {
            swapLocations = swapManager.recoverSwapLocations(this);
        } catch (final IOException ioe) {
            logger.error("Failed to determine whether or not any Swap Files exist for FlowFile Queue {}", getIdentifier());
            logger.error("", ioe);
            if (eventReporter != null) {
                eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to determine whether or not any Swap Files exist for FlowFile Queue " + getIdentifier() + "; see logs for more detials");
            }
            return null;
        }
        for (final String swapLocation : swapLocations) {
            try {
                final SwapSummary summary = swapManager.getSwapSummary(swapLocation);
                final QueueSize queueSize = summary.getQueueSize();
                final Long maxSwapRecordId = summary.getMaxFlowFileId();
                if (maxSwapRecordId != null) {
                    if (maxId == null || maxSwapRecordId > maxId) {
                        maxId = maxSwapRecordId;
                    }
                }
                swapFlowFileCount += queueSize.getObjectCount();
                swapByteCount += queueSize.getByteCount();
                resourceClaims.addAll(summary.getResourceClaims());
            } catch (final IOException ioe) {
                logger.error("Failed to recover FlowFiles from Swap File {}; the file appears to be corrupt", swapLocation, ioe.toString());
                logger.error("", ioe);
                if (eventReporter != null) {
                    eventReporter.reportEvent(Severity.ERROR, "FlowFile Swapping", "Failed to recover FlowFiles from Swap File " + swapLocation + "; the file appears to be corrupt. See logs for more details");
                }
            }
        }
        incrementSwapQueueSize(swapFlowFileCount, swapByteCount, swapLocations.size());
        this.swapLocations.addAll(swapLocations);
    } finally {
        writeLock.unlock("Recover Swap Files");
    }
    if (!swapLocations.isEmpty()) {
        final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
        logger.info("Recovered {} swap files for {} in {} millis", swapLocations.size(), this, millis);
    }
    return new StandardSwapSummary(new QueueSize(swapFlowFileCount, swapByteCount), maxId, resourceClaims);
}

Example 3 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class SchemaSwapDeserializer method getSwapSummary.

@Override
public SwapSummary getSwapSummary(final DataInputStream in, final String swapLocation, final ResourceClaimManager claimManager) throws IOException {
    final RecordSchema schema = RecordSchema.readFrom(in);
    final List<RecordField> summaryFields = schema.getField(SwapSchema.SWAP_SUMMARY).getSubFields();
    final RecordField summaryRecordField = new ComplexRecordField(SwapSchema.SWAP_SUMMARY, Repetition.EXACTLY_ONE, summaryFields);
    final RecordSchema summarySchema = new RecordSchema(Collections.singletonList(summaryRecordField));
    final Record summaryRecordParent = SchemaRecordReader.fromSchema(summarySchema).readRecord(in);
    final Record summaryRecord = (Record) summaryRecordParent.getFieldValue(SwapSchema.SWAP_SUMMARY);
    final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager);
    return swapSummary;
}

Example 4 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class SchemaSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException {
    schema.writeTo(out);
    long contentSize = 0L;
    long maxFlowFileId = -1L;
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final FlowFileRecord flowFile : toSwap) {
        contentSize += flowFile.getSize();
        if (flowFile.getId() > maxFlowFileId) {
            maxFlowFileId = flowFile.getId();
        }
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim != null) {
            resourceClaims.add(contentClaim.getResourceClaim());
        }
    }
    final QueueSize queueSize = new QueueSize(toSwap.size(), contentSize);
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
    final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
    final List<Record> flowFileRecords = toSwap.stream().map(flowFile -> new FlowFileRecordFieldMap(flowFile, flowFileSchema)).collect(Collectors.toList());
    // Create a simple record to hold the summary and the flowfile contents
    final RecordField summaryField = new SimpleRecordField(SwapSchema.SWAP_SUMMARY, FieldType.COMPLEX, Repetition.EXACTLY_ONE);
    final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields());
    final List<RecordField> fields = new ArrayList<>(2);
    fields.add(summaryField);
    fields.add(contentsField);
    final Map<RecordField, Object> swapFileMap = new LinkedHashMap<>();
    swapFileMap.put(summaryField, summaryRecord);
    swapFileMap.put(contentsField, flowFileRecords);
    final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
    final SchemaRecordWriter writer = new SchemaRecordWriter();
    writer.writeRecord(swapFileRecord, out);
    out.flush();
}

Example 5 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class FlowController method initializeFlow.

public void initializeFlow() throws IOException {
    writeLock.lock();
    try {
        // get all connections/queues and recover from swap files.
        final List<Connection> connections = getGroup(getRootGroupId()).findAllConnections();
        long maxIdFromSwapFiles = -1L;
        if (flowFileRepository.isVolatile()) {
            for (final Connection connection : connections) {
                final FlowFileQueue queue = connection.getFlowFileQueue();
                queue.purgeSwapFiles();
            }
        } else {
            for (final Connection connection : connections) {
                final FlowFileQueue queue = connection.getFlowFileQueue();
                final SwapSummary swapSummary = queue.recoverSwappedFlowFiles();
                if (swapSummary != null) {
                    final Long maxFlowFileId = swapSummary.getMaxFlowFileId();
                    if (maxFlowFileId != null && maxFlowFileId > maxIdFromSwapFiles) {
                        maxIdFromSwapFiles = maxFlowFileId;
                    }
                    for (final ResourceClaim resourceClaim : swapSummary.getResourceClaims()) {
                        resourceClaimManager.incrementClaimantCount(resourceClaim);
                    }
                }
            }
        }
        flowFileRepository.loadFlowFiles(this, maxIdFromSwapFiles + 1);
        // Begin expiring FlowFiles that are old
        final RepositoryContextFactory contextFactory = new RepositoryContextFactory(contentRepository, flowFileRepository, flowFileEventRepository, counterRepositoryRef.get(), provenanceRepository);
        processScheduler.scheduleFrameworkTask(new ExpireFlowFiles(this, contextFactory), "Expire FlowFiles", 30L, 30L, TimeUnit.SECONDS);
        // now that we've loaded the FlowFiles, this has restored our ContentClaims' states, so we can tell the
        // ContentRepository to purge superfluous files
        contentRepository.cleanup();
        for (final RemoteSiteListener listener : externalSiteListeners) {
            listener.start();
        }
        notifyComponentsConfigurationRestored();
        timerDrivenEngineRef.get().scheduleWithFixedDelay(new Runnable() {

            @Override
            public void run() {
                try {
                    updateRemoteProcessGroups();
                } catch (final Throwable t) {
                    LOG.warn("Unable to update Remote Process Groups due to " + t);
                    if (LOG.isDebugEnabled()) {
                        LOG.warn("", t);
                    }
                }
            }
        }, 0L, 30L, TimeUnit.SECONDS);
        timerDrivenEngineRef.get().scheduleWithFixedDelay(new Runnable() {

            @Override
            public void run() {
                final ProcessGroup rootGroup = getRootGroup();
                final List<ProcessGroup> allGroups = rootGroup.findAllProcessGroups();
                allGroups.add(rootGroup);
                for (final ProcessGroup group : allGroups) {
                    try {
                        group.synchronizeWithFlowRegistry(flowRegistryClient);
                    } catch (final Exception e) {
                        LOG.error("Failed to synchronize {} with Flow Registry", group, e);
                    }
                }
            }
        }, 5, 60, TimeUnit.SECONDS);
        initialized.set(true);
    } finally {
        writeLock.unlock();
    }
}