Examples with VersionedFlowSnapshot org.apache.nifi.registry.flow.VersionedFlowSnapshot used on opensource projects

Example 21 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi-registry by apache.

the class UnsecuredNiFiRegistryClientIT method testNiFiRegistryClient.

@Test
public void testNiFiRegistryClient() throws IOException, NiFiRegistryException {
    // ---------------------- TEST BUCKETS --------------------------//
    final BucketClient bucketClient = client.getBucketClient();
    // create buckets
    final int numBuckets = 10;
    final List<Bucket> createdBuckets = new ArrayList<>();
    for (int i = 0; i < numBuckets; i++) {
        final Bucket createdBucket = createBucket(bucketClient, i);
        LOGGER.info("Created bucket # " + i + " with id " + createdBucket.getIdentifier());
        createdBuckets.add(createdBucket);
    }
    // get each bucket
    for (final Bucket bucket : createdBuckets) {
        final Bucket retrievedBucket = bucketClient.get(bucket.getIdentifier());
        Assert.assertNotNull(retrievedBucket);
        LOGGER.info("Retrieved bucket " + retrievedBucket.getIdentifier());
    }
    // final Bucket nonExistentBucket = bucketClient.get("does-not-exist");
    // Assert.assertNull(nonExistentBucket);
    // get bucket fields
    final Fields bucketFields = bucketClient.getFields();
    Assert.assertNotNull(bucketFields);
    LOGGER.info("Retrieved bucket fields, size = " + bucketFields.getFields().size());
    Assert.assertTrue(bucketFields.getFields().size() > 0);
    // get all buckets
    final List<Bucket> allBuckets = bucketClient.getAll();
    LOGGER.info("Retrieved buckets, size = " + allBuckets.size());
    Assert.assertEquals(numBuckets, allBuckets.size());
    allBuckets.stream().forEach(b -> System.out.println("Retrieve bucket " + b.getIdentifier()));
    // update each bucket
    for (final Bucket bucket : createdBuckets) {
        final Bucket bucketUpdate = new Bucket();
        bucketUpdate.setIdentifier(bucket.getIdentifier());
        bucketUpdate.setDescription(bucket.getDescription() + " UPDATE");
        final Bucket updatedBucket = bucketClient.update(bucketUpdate);
        Assert.assertNotNull(updatedBucket);
        LOGGER.info("Updated bucket " + updatedBucket.getIdentifier());
    }
    // ---------------------- TEST FLOWS --------------------------//
    final FlowClient flowClient = client.getFlowClient();
    // create flows
    final Bucket flowsBucket = createdBuckets.get(0);
    final VersionedFlow flow1 = createFlow(flowClient, flowsBucket, 1);
    LOGGER.info("Created flow # 1 with id " + flow1.getIdentifier());
    final VersionedFlow flow2 = createFlow(flowClient, flowsBucket, 2);
    LOGGER.info("Created flow # 2 with id " + flow2.getIdentifier());
    // get flow
    final VersionedFlow retrievedFlow1 = flowClient.get(flowsBucket.getIdentifier(), flow1.getIdentifier());
    Assert.assertNotNull(retrievedFlow1);
    LOGGER.info("Retrieved flow # 1 with id " + retrievedFlow1.getIdentifier());
    final VersionedFlow retrievedFlow2 = flowClient.get(flowsBucket.getIdentifier(), flow2.getIdentifier());
    Assert.assertNotNull(retrievedFlow2);
    LOGGER.info("Retrieved flow # 2 with id " + retrievedFlow2.getIdentifier());
    // update flows
    final VersionedFlow flow1Update = new VersionedFlow();
    flow1Update.setIdentifier(flow1.getIdentifier());
    flow1Update.setName(flow1.getName() + " UPDATED");
    final VersionedFlow updatedFlow1 = flowClient.update(flowsBucket.getIdentifier(), flow1Update);
    Assert.assertNotNull(updatedFlow1);
    LOGGER.info("Updated flow # 1 with id " + updatedFlow1.getIdentifier());
    // get flow fields
    final Fields flowFields = flowClient.getFields();
    Assert.assertNotNull(flowFields);
    LOGGER.info("Retrieved flow fields, size = " + flowFields.getFields().size());
    Assert.assertTrue(flowFields.getFields().size() > 0);
    // get flows in bucket
    final List<VersionedFlow> flowsInBucket = flowClient.getByBucket(flowsBucket.getIdentifier());
    Assert.assertNotNull(flowsInBucket);
    Assert.assertEquals(2, flowsInBucket.size());
    flowsInBucket.stream().forEach(f -> LOGGER.info("Flow in bucket, flow id " + f.getIdentifier()));
    // ---------------------- TEST SNAPSHOTS --------------------------//
    final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient();
    // create snapshots
    final VersionedFlow snapshotFlow = flow1;
    final VersionedFlowSnapshot snapshot1 = createSnapshot(snapshotClient, snapshotFlow, 1);
    LOGGER.info("Created snapshot # 1 with version " + snapshot1.getSnapshotMetadata().getVersion());
    final VersionedFlowSnapshot snapshot2 = createSnapshot(snapshotClient, snapshotFlow, 2);
    LOGGER.info("Created snapshot # 2 with version " + snapshot2.getSnapshotMetadata().getVersion());
    // get snapshot
    final VersionedFlowSnapshot retrievedSnapshot1 = snapshotClient.get(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier(), 1);
    Assert.assertNotNull(retrievedSnapshot1);
    Assert.assertFalse(retrievedSnapshot1.isLatest());
    LOGGER.info("Retrieved snapshot # 1 with version " + retrievedSnapshot1.getSnapshotMetadata().getVersion());
    final VersionedFlowSnapshot retrievedSnapshot2 = snapshotClient.get(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier(), 2);
    Assert.assertNotNull(retrievedSnapshot2);
    Assert.assertTrue(retrievedSnapshot2.isLatest());
    LOGGER.info("Retrieved snapshot # 2 with version " + retrievedSnapshot2.getSnapshotMetadata().getVersion());
    // get latest
    final VersionedFlowSnapshot retrievedSnapshotLatest = snapshotClient.getLatest(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier());
    Assert.assertNotNull(retrievedSnapshotLatest);
    Assert.assertEquals(snapshot2.getSnapshotMetadata().getVersion(), retrievedSnapshotLatest.getSnapshotMetadata().getVersion());
    Assert.assertTrue(retrievedSnapshotLatest.isLatest());
    LOGGER.info("Retrieved latest snapshot with version " + retrievedSnapshotLatest.getSnapshotMetadata().getVersion());
    // get metadata
    final List<VersionedFlowSnapshotMetadata> retrievedMetadata = snapshotClient.getSnapshotMetadata(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier());
    Assert.assertNotNull(retrievedMetadata);
    Assert.assertEquals(2, retrievedMetadata.size());
    Assert.assertEquals(2, retrievedMetadata.get(0).getVersion());
    Assert.assertEquals(1, retrievedMetadata.get(1).getVersion());
    retrievedMetadata.stream().forEach(s -> LOGGER.info("Retrieved snapshot metadata " + s.getVersion()));
    // get latest metadata
    final VersionedFlowSnapshotMetadata latestMetadata = snapshotClient.getLatestMetadata(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier());
    Assert.assertNotNull(latestMetadata);
    Assert.assertEquals(2, latestMetadata.getVersion());
    // get latest metadata that doesn't exist
    try {
        snapshotClient.getLatestMetadata(snapshotFlow.getBucketIdentifier(), "DOES-NOT-EXIST");
        Assert.fail("Should have thrown exception");
    } catch (NiFiRegistryException nfe) {
        Assert.assertEquals("Error retrieving latest snapshot metadata: The specified flow ID does not exist in this bucket.", nfe.getMessage());
    }
    // ---------------------- TEST ITEMS --------------------------//
    final ItemsClient itemsClient = client.getItemsClient();
    // get fields
    final Fields itemFields = itemsClient.getFields();
    Assert.assertNotNull(itemFields.getFields());
    Assert.assertTrue(itemFields.getFields().size() > 0);
    // get all items
    final List<BucketItem> allItems = itemsClient.getAll();
    Assert.assertEquals(2, allItems.size());
    allItems.stream().forEach(i -> Assert.assertNotNull(i.getBucketName()));
    allItems.stream().forEach(i -> LOGGER.info("All items, item " + i.getIdentifier()));
    // get items for bucket
    final List<BucketItem> bucketItems = itemsClient.getByBucket(flowsBucket.getIdentifier());
    Assert.assertEquals(2, bucketItems.size());
    allItems.stream().forEach(i -> Assert.assertNotNull(i.getBucketName()));
    bucketItems.stream().forEach(i -> LOGGER.info("Items in bucket, item " + i.getIdentifier()));
    // ----------------------- TEST DIFF ---------------------------//
    final VersionedFlowSnapshot snapshot3 = buildSnapshot(snapshotFlow, 3);
    final VersionedProcessGroup newlyAddedPG = new VersionedProcessGroup();
    newlyAddedPG.setIdentifier("new-pg");
    newlyAddedPG.setName("NEW Process Group");
    snapshot3.getFlowContents().getProcessGroups().add(newlyAddedPG);
    snapshotClient.create(snapshot3);
    VersionedFlowDifference diff = flowClient.diff(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier(), 3, 2);
    Assert.assertNotNull(diff);
    Assert.assertEquals(1, diff.getComponentDifferenceGroups().size());
    // ---------------------- DELETE DATA --------------------------//
    final VersionedFlow deletedFlow1 = flowClient.delete(flowsBucket.getIdentifier(), flow1.getIdentifier());
    Assert.assertNotNull(deletedFlow1);
    LOGGER.info("Deleted flow " + deletedFlow1.getIdentifier());
    final VersionedFlow deletedFlow2 = flowClient.delete(flowsBucket.getIdentifier(), flow2.getIdentifier());
    Assert.assertNotNull(deletedFlow2);
    LOGGER.info("Deleted flow " + deletedFlow2.getIdentifier());
    // delete each bucket
    for (final Bucket bucket : createdBuckets) {
        final Bucket deletedBucket = bucketClient.delete(bucket.getIdentifier());
        Assert.assertNotNull(deletedBucket);
        LOGGER.info("Deleted bucket " + deletedBucket.getIdentifier());
    }
    Assert.assertEquals(0, bucketClient.getAll().size());
    LOGGER.info("!!! SUCCESS !!!");
}

Example 22 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi-registry by apache.

the class BucketFlowResource method getLatestFlowVersion.

@GET
@Path("{flowId}/versions/latest")
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Get the latest version of a flow", response = VersionedFlowSnapshot.class, extensions = { @Extension(name = "access-policy", properties = { @ExtensionProperty(name = "action", value = "read"), @ExtensionProperty(name = "resource", value = "/buckets/{bucketId}") }) })
@ApiResponses({ @ApiResponse(code = 401, message = HttpStatusMessages.MESSAGE_401), @ApiResponse(code = 403, message = HttpStatusMessages.MESSAGE_403), @ApiResponse(code = 404, message = HttpStatusMessages.MESSAGE_404), @ApiResponse(code = 409, message = HttpStatusMessages.MESSAGE_409) })
public Response getLatestFlowVersion(@PathParam("bucketId") @ApiParam("The bucket identifier") final String bucketId, @PathParam("flowId") @ApiParam("The flow identifier") final String flowId) {
    authorizeBucketAccess(RequestAction.READ, bucketId);
    final VersionedFlowSnapshotMetadata latestMetadata = registryService.getLatestFlowSnapshotMetadata(bucketId, flowId);
    final VersionedFlowSnapshot lastSnapshot = registryService.getFlowSnapshot(bucketId, flowId, latestMetadata.getVersion());
    populateLinksAndPermissions(lastSnapshot);
    return Response.status(Response.Status.OK).entity(lastSnapshot).build();
}

Example 23 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi-registry by apache.

the class TestRestAPI method createSnapshot.

private static VersionedFlowSnapshot createSnapshot(Client client, VersionedFlow flow, int num) {
    final VersionedFlowSnapshotMetadata snapshotMetadata1 = new VersionedFlowSnapshotMetadata();
    snapshotMetadata1.setBucketIdentifier(flow.getBucketIdentifier());
    snapshotMetadata1.setFlowIdentifier(flow.getIdentifier());
    snapshotMetadata1.setVersion(num);
    snapshotMetadata1.setComments("This is snapshot #" + num);
    final VersionedProcessGroup snapshotContents1 = new VersionedProcessGroup();
    snapshotContents1.setIdentifier("pg1");
    snapshotContents1.setName("Process Group 1");
    final VersionedFlowSnapshot snapshot1 = new VersionedFlowSnapshot();
    snapshot1.setSnapshotMetadata(snapshotMetadata1);
    snapshot1.setFlowContents(snapshotContents1);
    final VersionedFlowSnapshot createdSnapshot = client.target(REGISTRY_API_BUCKETS_URL).path("{bucketId}/flows/{flowId}/versions").resolveTemplate("bucketId", flow.getBucketIdentifier()).resolveTemplate("flowId", flow.getIdentifier()).request().post(Entity.entity(snapshot1, MediaType.APPLICATION_JSON_TYPE), VersionedFlowSnapshot.class);
    return createdSnapshot;
}

Example 24 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi-minifi by apache.

the class ConfigMain method transformVersionedFlowSnapshotToSchema.

public static ConfigSchema transformVersionedFlowSnapshotToSchema(InputStream source) throws IOException {
    try {
        final ObjectMapper objectMapper = new ObjectMapper();
        objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
        objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory()));
        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
        final VersionedFlowSnapshot versionedFlowSnapshot = objectMapper.readValue(source, VersionedFlowSnapshot.class);
        return transformVersionedFlowSnapshotToSchema(versionedFlowSnapshot);
    } finally {
        source.close();
    }
}

Example 25 with VersionedFlowSnapshot

use of org.apache.nifi.registry.flow.VersionedFlowSnapshot in project nifi by apache.

the class StandardNiFiServiceFacade method getComponentsAffectedByVersionChange.

@Override
public Set<AffectedComponentEntity> getComponentsAffectedByVersionChange(final String processGroupId, final VersionedFlowSnapshot updatedSnapshot, final NiFiUser user) {
    final ProcessGroup group = processGroupDAO.getProcessGroup(processGroupId);
    final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper();
    final VersionedProcessGroup localContents = mapper.mapProcessGroup(group, controllerFacade.getControllerServiceProvider(), flowRegistryClient, true);
    final ComparableDataFlow localFlow = new StandardComparableDataFlow("Local Flow", localContents);
    final ComparableDataFlow proposedFlow = new StandardComparableDataFlow("Versioned Flow", updatedSnapshot.getFlowContents());
    final Set<String> ancestorGroupServiceIds = getAncestorGroupServiceIds(group);
    final FlowComparator flowComparator = new StandardFlowComparator(localFlow, proposedFlow, ancestorGroupServiceIds, new StaticDifferenceDescriptor());
    final FlowComparison comparison = flowComparator.compare();
    final Set<AffectedComponentEntity> affectedComponents = comparison.getDifferences().stream().filter(// components that are added are not components that will be affected in the local flow.
    difference -> difference.getDifferenceType() != DifferenceType.COMPONENT_ADDED).filter(difference -> difference.getDifferenceType() != DifferenceType.BUNDLE_CHANGED).filter(FlowDifferenceFilters.FILTER_ADDED_REMOVED_REMOTE_PORTS).map(difference -> {
        final VersionedComponent localComponent = difference.getComponentA();
        final String state;
        switch(localComponent.getComponentType()) {
            case CONTROLLER_SERVICE:
                final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
                state = controllerServiceDAO.getControllerService(serviceId).getState().name();
                break;
            case PROCESSOR:
                final String processorId = ((InstantiatedVersionedProcessor) localComponent).getInstanceId();
                state = processorDAO.getProcessor(processorId).getPhysicalScheduledState().name();
                break;
            case REMOTE_INPUT_PORT:
                final InstantiatedVersionedRemoteGroupPort inputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
                state = remoteProcessGroupDAO.getRemoteProcessGroup(inputPort.getInstanceGroupId()).getInputPort(inputPort.getInstanceId()).getScheduledState().name();
                break;
            case REMOTE_OUTPUT_PORT:
                final InstantiatedVersionedRemoteGroupPort outputPort = (InstantiatedVersionedRemoteGroupPort) localComponent;
                state = remoteProcessGroupDAO.getRemoteProcessGroup(outputPort.getInstanceGroupId()).getOutputPort(outputPort.getInstanceId()).getScheduledState().name();
                break;
            default:
                state = null;
                break;
        }
        return createAffectedComponentEntity((InstantiatedVersionedComponent) localComponent, localComponent.getComponentType().name(), state, user);
    }).collect(Collectors.toCollection(HashSet::new));
    for (final FlowDifference difference : comparison.getDifferences()) {
        // Ignore these as local differences for now because we can't do anything with it
        if (difference.getDifferenceType() == DifferenceType.BUNDLE_CHANGED) {
            continue;
        }
        // Ignore differences for adding remote ports
        if (FlowDifferenceFilters.isAddedOrRemovedRemotePort(difference)) {
            continue;
        }
        final VersionedComponent localComponent = difference.getComponentA();
        if (localComponent == null) {
            continue;
        }
        // If any Process Group is removed, consider all components below that Process Group as an affected component
        if (difference.getDifferenceType() == DifferenceType.COMPONENT_REMOVED && localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.PROCESS_GROUP) {
            final String localGroupId = ((InstantiatedVersionedProcessGroup) localComponent).getInstanceId();
            final ProcessGroup localGroup = processGroupDAO.getProcessGroup(localGroupId);
            localGroup.findAllProcessors().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllFunnels().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllInputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllOutputPorts().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllRemoteProcessGroups().stream().flatMap(rpg -> Stream.concat(rpg.getInputPorts().stream(), rpg.getOutputPorts().stream())).map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
            localGroup.findAllControllerServices().stream().map(comp -> createAffectedComponentEntity(comp, user)).forEach(affectedComponents::add);
        }
        if (localComponent.getComponentType() == org.apache.nifi.registry.flow.ComponentType.CONTROLLER_SERVICE) {
            final String serviceId = ((InstantiatedVersionedControllerService) localComponent).getInstanceId();
            final ControllerServiceNode serviceNode = controllerServiceDAO.getControllerService(serviceId);
            final List<ControllerServiceNode> referencingServices = serviceNode.getReferences().findRecursiveReferences(ControllerServiceNode.class);
            for (final ControllerServiceNode referencingService : referencingServices) {
                affectedComponents.add(createAffectedComponentEntity(referencingService, user));
            }
            final List<ProcessorNode> referencingProcessors = serviceNode.getReferences().findRecursiveReferences(ProcessorNode.class);
            for (final ProcessorNode referencingProcessor : referencingProcessors) {
                affectedComponents.add(createAffectedComponentEntity(referencingProcessor, user));
            }
        }
    }
    // Create a map of all connectable components by versioned component ID to the connectable component itself
    final Map<String, List<Connectable>> connectablesByVersionId = new HashMap<>();
    mapToConnectableId(group.findAllFunnels(), connectablesByVersionId);
    mapToConnectableId(group.findAllInputPorts(), connectablesByVersionId);
    mapToConnectableId(group.findAllOutputPorts(), connectablesByVersionId);
    mapToConnectableId(group.findAllProcessors(), connectablesByVersionId);
    final List<RemoteGroupPort> remotePorts = new ArrayList<>();
    for (final RemoteProcessGroup rpg : group.findAllRemoteProcessGroups()) {
        remotePorts.addAll(rpg.getInputPorts());
        remotePorts.addAll(rpg.getOutputPorts());
    }
    mapToConnectableId(remotePorts, connectablesByVersionId);
    // and the destination (if it exists in the flow currently).
    for (final FlowDifference difference : comparison.getDifferences()) {
        VersionedComponent component = difference.getComponentA();
        if (component == null) {
            component = difference.getComponentB();
        }
        if (component.getComponentType() != org.apache.nifi.registry.flow.ComponentType.CONNECTION) {
            continue;
        }
        final VersionedConnection connection = (VersionedConnection) component;
        final String sourceVersionedId = connection.getSource().getId();
        final List<Connectable> sources = connectablesByVersionId.get(sourceVersionedId);
        if (sources != null) {
            for (final Connectable source : sources) {
                affectedComponents.add(createAffectedComponentEntity(source, user));
            }
        }
        final String destinationVersionId = connection.getDestination().getId();
        final List<Connectable> destinations = connectablesByVersionId.get(destinationVersionId);
        if (destinations != null) {
            for (final Connectable destination : destinations) {
                affectedComponents.add(createAffectedComponentEntity(destination, user));
            }
        }
    }
    return affectedComponents;
}