Examples with SwapSummary org.apache.nifi.controller.repository.SwapSummary used on opensource projects

Example 6 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testRoundTripSerializeDeserializeSummary.

@Test
public void testRoundTripSerializeDeserializeSummary() throws IOException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final ResourceClaim firstResourceClaim = resourceClaimManager.newResourceClaim("container", "section", "id", true, false);
    resourceClaimManager.incrementClaimantCount(firstResourceClaim);
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    long size = 0L;
    final ContentClaim firstClaim = MockFlowFile.createContentClaim("id", resourceClaimManager);
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = i < 2 ? new MockFlowFile(attrs, i, firstClaim) : new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
        size += i;
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    final SwapSerializer serializer = new SchemaSwapSerializer();
    try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
        serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
    }
    final SwapDeserializer deserializer = new SchemaSwapDeserializer();
    final SwapSummary swapSummary;
    try (final FileInputStream fis = new FileInputStream(swapFile);
        final DataInputStream dis = new DataInputStream(fis)) {
        swapSummary = deserializer.getSwapSummary(dis, swapLocation, resourceClaimManager);
    }
    assertEquals(10000, swapSummary.getQueueSize().getObjectCount());
    assertEquals(size, swapSummary.getQueueSize().getByteCount());
    assertEquals(9999, swapSummary.getMaxFlowFileId().intValue());
    final List<ResourceClaim> resourceClaims = swapSummary.getResourceClaims();
    assertEquals(10000, resourceClaims.size());
    assertFalse(resourceClaims.stream().anyMatch(claim -> claim == null));
    assertEquals(2, resourceClaims.stream().filter(claim -> claim.getId().equals("id")).collect(Collectors.counting()).intValue());
    final Set<ResourceClaim> uniqueClaims = new HashSet<>(resourceClaims);
    assertEquals(9999, uniqueClaims.size());
}

Example 7 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class SchemaSwapDeserializer method deserializeFlowFiles.

@Override
@SuppressWarnings("unchecked")
public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException {
    final RecordSchema schema = RecordSchema.readFrom(in);
    final SchemaRecordReader reader = SchemaRecordReader.fromSchema(schema);
    final Record parentRecord = reader.readRecord(in);
    final List<Record> flowFileRecords = (List<Record>) parentRecord.getFieldValue(SwapSchema.FLOWFILE_CONTENTS);
    final List<FlowFileRecord> flowFiles = new ArrayList<>(flowFileRecords.size());
    for (final Record record : flowFileRecords) {
        flowFiles.add(FlowFileRecordFieldMap.getFlowFile(record, claimManager));
    }
    final Record summaryRecord = (Record) parentRecord.getFieldValue(SwapSchema.SWAP_SUMMARY);
    final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager);
    return new StandardSwapContents(swapSummary, flowFiles);
}

Example 8 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class SimpleSwapDeserializer method deserializeFlowFiles.

@Override
public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException {
    final int swapEncodingVersion = in.readInt();
    if (swapEncodingVersion > SWAP_ENCODING_VERSION) {
        throw new IOException("Cannot swap FlowFiles in from SwapFile because the encoding version is " + swapEncodingVersion + ", which is too new (expecting " + SWAP_ENCODING_VERSION + " or less)");
    }
    // Connection ID
    final String connectionId = in.readUTF();
    if (!connectionId.equals(queue.getIdentifier())) {
        throw new IllegalArgumentException("Cannot deserialize FlowFiles from Swap File at location " + swapLocation + " because those FlowFiles belong to Connection with ID " + connectionId + " and an attempt was made to swap them into a Connection with ID " + queue.getIdentifier());
    }
    int numRecords = 0;
    long contentSize = 0L;
    Long maxRecordId = null;
    try {
        numRecords = in.readInt();
        // Content Size
        contentSize = in.readLong();
        if (swapEncodingVersion > 7) {
            // Max Record ID
            maxRecordId = in.readLong();
        }
    } catch (final EOFException eof) {
        final QueueSize queueSize = new QueueSize(numRecords, contentSize);
        final SwapSummary summary = new StandardSwapSummary(queueSize, maxRecordId, Collections.emptyList());
        final SwapContents partialContents = new StandardSwapContents(summary, Collections.emptyList());
        throw new IncompleteSwapFileException(swapLocation, partialContents);
    }
    final QueueSize queueSize = new QueueSize(numRecords, contentSize);
    return deserializeFlowFiles(in, queueSize, maxRecordId, swapEncodingVersion, claimManager, swapLocation);
}

Example 9 with SwapSummary

use of org.apache.nifi.controller.repository.SwapSummary in project nifi by apache.

the class SimpleSwapDeserializer method deserializeFlowFiles.

private static SwapContents deserializeFlowFiles(final DataInputStream in, final QueueSize queueSize, final Long maxRecordId, final int serializationVersion, final ResourceClaimManager claimManager, final String location) throws IOException {
    final List<FlowFileRecord> flowFiles = new ArrayList<>(queueSize.getObjectCount());
    final List<ResourceClaim> resourceClaims = new ArrayList<>(queueSize.getObjectCount());
    Long maxId = maxRecordId;
    for (int i = 0; i < queueSize.getObjectCount(); i++) {
        try {
            // legacy encoding had an "action" because it used to be couple with FlowFile Repository code
            if (serializationVersion < 3) {
                final int action = in.read();
                if (action != 1) {
                    throw new IOException("Swap File is version " + serializationVersion + " but did not contain a 'UPDATE' record type");
                }
            }
            final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder();
            final long recordId = in.readLong();
            if (maxId == null || recordId > maxId) {
                maxId = recordId;
            }
            ffBuilder.id(recordId);
            ffBuilder.entryDate(in.readLong());
            if (serializationVersion > 1) {
                // Lineage information was added in version 2
                if (serializationVersion < 10) {
                    final int numLineageIdentifiers = in.readInt();
                    for (int lineageIdIdx = 0; lineageIdIdx < numLineageIdentifiers; lineageIdIdx++) {
                        // skip each identifier
                        in.readUTF();
                    }
                }
                // version 9 adds in a 'lineage start index'
                final long lineageStartDate = in.readLong();
                final long lineageStartIndex;
                if (serializationVersion > 8) {
                    lineageStartIndex = in.readLong();
                } else {
                    lineageStartIndex = 0L;
                }
                ffBuilder.lineageStart(lineageStartDate, lineageStartIndex);
                if (serializationVersion > 5) {
                    // Version 9 adds in a 'queue date index'
                    final long lastQueueDate = in.readLong();
                    final long queueDateIndex;
                    if (serializationVersion > 8) {
                        queueDateIndex = in.readLong();
                    } else {
                        queueDateIndex = 0L;
                    }
                    ffBuilder.lastQueued(lastQueueDate, queueDateIndex);
                }
            }
            ffBuilder.size(in.readLong());
            if (serializationVersion < 3) {
                // connection Id
                readString(in);
            }
            final boolean hasClaim = in.readBoolean();
            ResourceClaim resourceClaim = null;
            if (hasClaim) {
                final String claimId;
                if (serializationVersion < 5) {
                    claimId = String.valueOf(in.readLong());
                } else {
                    claimId = in.readUTF();
                }
                final String container = in.readUTF();
                final String section = in.readUTF();
                final long resourceOffset;
                final long resourceLength;
                if (serializationVersion < 6) {
                    resourceOffset = 0L;
                    resourceLength = -1L;
                } else {
                    resourceOffset = in.readLong();
                    resourceLength = in.readLong();
                }
                final long claimOffset = in.readLong();
                final boolean lossTolerant;
                if (serializationVersion >= 4) {
                    lossTolerant = in.readBoolean();
                } else {
                    lossTolerant = false;
                }
                resourceClaim = claimManager.getResourceClaim(container, section, claimId);
                if (resourceClaim == null) {
                    logger.error("Swap file indicates that FlowFile was referencing Resource Claim at container={}, section={}, claimId={}, " + "but this Resource Claim cannot be found! Will create a temporary Resource Claim, but this may affect the framework's " + "ability to properly clean up this resource", container, section, claimId);
                    resourceClaim = claimManager.newResourceClaim(container, section, claimId, lossTolerant, true);
                }
                final StandardContentClaim claim = new StandardContentClaim(resourceClaim, resourceOffset);
                claim.setLength(resourceLength);
                ffBuilder.contentClaim(claim);
                ffBuilder.contentClaimOffset(claimOffset);
            }
            boolean attributesChanged = true;
            if (serializationVersion < 3) {
                attributesChanged = in.readBoolean();
            }
            if (attributesChanged) {
                final int numAttributes = in.readInt();
                for (int j = 0; j < numAttributes; j++) {
                    final String key = readString(in);
                    final String value = readString(in);
                    ffBuilder.addAttribute(key, value);
                }
            }
            final FlowFileRecord record = ffBuilder.build();
            if (resourceClaim != null) {
                resourceClaims.add(resourceClaim);
            }
            flowFiles.add(record);
        } catch (final EOFException eof) {
            final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxId, resourceClaims);
            final SwapContents partialContents = new StandardSwapContents(swapSummary, flowFiles);
            throw new IncompleteSwapFileException(location, partialContents);
        }
    }
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxId, resourceClaims);
    return new StandardSwapContents(swapSummary, flowFiles);
}