Examples with VersionedProcessGroup org.apache.nifi.registry.flow.VersionedProcessGroup used on opensource projects

Example 16 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi-registry by apache.

the class TestRegistryService method createSnapshot.

// ---------------------- Test VersionedFlowSnapshot methods ---------------------------------------------
private VersionedFlowSnapshot createSnapshot() {
    final VersionedFlowSnapshotMetadata snapshotMetadata = new VersionedFlowSnapshotMetadata();
    snapshotMetadata.setFlowIdentifier("flow1");
    snapshotMetadata.setVersion(1);
    snapshotMetadata.setComments("This is the first snapshot");
    snapshotMetadata.setBucketIdentifier("b1");
    snapshotMetadata.setAuthor("user1");
    final VersionedProcessGroup processGroup = new VersionedProcessGroup();
    processGroup.setIdentifier("pg1");
    processGroup.setName("My Process Group");
    final VersionedFlowSnapshot snapshot = new VersionedFlowSnapshot();
    snapshot.setSnapshotMetadata(snapshotMetadata);
    snapshot.setFlowContents(processGroup);
    return snapshot;
}

Example 17 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi-registry by apache.

the class TestRegistryService method createVersionedProcessGroupB.

private VersionedProcessGroup createVersionedProcessGroupB() {
    VersionedProcessGroup updated = createVersionedProcessGroupA();
    // remove a process group
    updated.getProcessGroups().removeIf(pg -> pg.getIdentifier().equals("ID-pg1"));
    // change the name of a processor
    updated.getProcessors().stream().forEach(p -> p.setPenaltyDuration(p.getName().equals("ProcessorFirstV1") ? "1" : "2"));
    updated.getProcessors().stream().forEach(p -> p.setName(p.getName().equals("ProcessorFirstV1") ? "ProcessorFirstV2" : p.getName()));
    return updated;
}

Example 18 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi-registry by apache.

the class RegistryService method getFlowDiff.

/**
 * Returns the differences between two specified versions of a flow.
 *
 * @param bucketIdentifier the id of the bucket the flow exists in
 * @param flowIdentifier the flow to be examined
 * @param versionA the first version of the comparison
 * @param versionB the second version of the comparison
 * @return The differences between two specified versions, grouped by component.
 */
public VersionedFlowDifference getFlowDiff(final String bucketIdentifier, final String flowIdentifier, final Integer versionA, final Integer versionB) {
    if (StringUtils.isBlank(bucketIdentifier)) {
        throw new IllegalArgumentException("Bucket identifier cannot be null or blank");
    }
    if (StringUtils.isBlank(flowIdentifier)) {
        throw new IllegalArgumentException("Flow identifier cannot be null or blank");
    }
    if (versionA == null || versionB == null) {
        throw new IllegalArgumentException("Version cannot be null or blank");
    }
    // older version is always the lower, regardless of the order supplied
    final Integer older = Math.min(versionA, versionB);
    final Integer newer = Math.max(versionA, versionB);
    readLock.lock();
    try {
        // Get the content for both versions of the flow
        final byte[] serializedSnapshotA = flowPersistenceProvider.getFlowContent(bucketIdentifier, flowIdentifier, older);
        if (serializedSnapshotA == null || serializedSnapshotA.length == 0) {
            throw new IllegalStateException("No serialized content found for snapshot with flow identifier " + flowIdentifier + " and version " + older);
        }
        final byte[] serializedSnapshotB = flowPersistenceProvider.getFlowContent(bucketIdentifier, flowIdentifier, newer);
        if (serializedSnapshotB == null || serializedSnapshotB.length == 0) {
            throw new IllegalStateException("No serialized content found for snapshot with flow identifier " + flowIdentifier + " and version " + newer);
        }
        // deserialize the contents
        final InputStream inputA = new ByteArrayInputStream(serializedSnapshotA);
        final VersionedProcessGroup flowContentsA = processGroupSerializer.deserialize(inputA);
        final InputStream inputB = new ByteArrayInputStream(serializedSnapshotB);
        final VersionedProcessGroup flowContentsB = processGroupSerializer.deserialize(inputB);
        final ComparableDataFlow comparableFlowA = new StandardComparableDataFlow(String.format("Version %d", older), flowContentsA);
        final ComparableDataFlow comparableFlowB = new StandardComparableDataFlow(String.format("Version %d", newer), flowContentsB);
        // Compare the two versions of the flow
        final FlowComparator flowComparator = new StandardFlowComparator(comparableFlowA, comparableFlowB, null, new ConciseEvolvingDifferenceDescriptor());
        final FlowComparison flowComparison = flowComparator.compare();
        VersionedFlowDifference result = new VersionedFlowDifference();
        result.setBucketId(bucketIdentifier);
        result.setFlowId(flowIdentifier);
        result.setVersionA(older);
        result.setVersionB(newer);
        Set<ComponentDifferenceGroup> differenceGroups = getStringComponentDifferenceGroupMap(flowComparison.getDifferences());
        result.setComponentDifferenceGroups(differenceGroups);
        return result;
    } finally {
        readLock.unlock();
    }
}

Example 19 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi-registry by apache.

the class FlowsIT method testCreateFlowVersionGetFlowVersion.

@Test
public void testCreateFlowVersionGetFlowVersion() throws Exception {
    // Given: an empty Bucket "3" (see FlowsIT.sql) with a newly created flow
    long testStartTime = System.currentTimeMillis();
    final String bucketId = "2";
    final VersionedFlow flow = new VersionedFlow();
    flow.setBucketIdentifier(bucketId);
    flow.setName("Test Flow for creating snapshots");
    flow.setDescription("This is a randomly named flow created by an integration test for the purpose of holding snapshots.");
    final VersionedFlow createdFlow = client.target(createURL("buckets/{bucketId}/flows")).resolveTemplate("bucketId", bucketId).request().post(Entity.entity(flow, MediaType.APPLICATION_JSON), VersionedFlow.class);
    final String flowId = createdFlow.getIdentifier();
    // When: an initial flow snapshot is created *without* a version
    final VersionedFlowSnapshotMetadata flowSnapshotMetadata = new VersionedFlowSnapshotMetadata();
    flowSnapshotMetadata.setBucketIdentifier("2");
    flowSnapshotMetadata.setFlowIdentifier(flowId);
    flowSnapshotMetadata.setComments("This is snapshot 1, created by an integration test.");
    final VersionedFlowSnapshot flowSnapshot = new VersionedFlowSnapshot();
    flowSnapshot.setSnapshotMetadata(flowSnapshotMetadata);
    // an empty root process group
    flowSnapshot.setFlowContents(new VersionedProcessGroup());
    WebTarget clientRequestTarget = client.target(createURL("buckets/{bucketId}/flows/{flowId}/versions")).resolveTemplate("bucketId", bucketId).resolveTemplate("flowId", flowId);
    final Response response = clientRequestTarget.request().post(Entity.entity(flowSnapshot, MediaType.APPLICATION_JSON), Response.class);
    // Then: an error is returned because version != 1
    assertEquals(400, response.getStatus());
    // But When: an initial flow snapshot is created with version == 1
    flowSnapshot.getSnapshotMetadata().setVersion(1);
    final VersionedFlowSnapshot createdFlowSnapshot = clientRequestTarget.request().post(Entity.entity(flowSnapshot, MediaType.APPLICATION_JSON), VersionedFlowSnapshot.class);
    // Then: the server returns the created flow snapshot, with server-set fields populated correctly :)
    assertFlowSnapshotsEqual(flowSnapshot, createdFlowSnapshot, false);
    // both server and client in same JVM, so there shouldn't be skew
    assertTrue(createdFlowSnapshot.getSnapshotMetadata().getTimestamp() - testStartTime > 0L);
    assertEquals("anonymous", createdFlowSnapshot.getSnapshotMetadata().getAuthor());
    assertNotNull(createdFlowSnapshot.getSnapshotMetadata().getLink());
    assertNotNull(createdFlowSnapshot.getSnapshotMetadata().getLink().getUri());
    assertNotNull(createdFlowSnapshot.getFlow());
    assertEquals(1, createdFlowSnapshot.getFlow().getVersionCount());
    assertNotNull(createdFlowSnapshot.getBucket());
    // And when .../flows/{id}/versions is queried, then the newly created flow snapshot is returned in the list
    final VersionedFlowSnapshotMetadata[] versionedFlowSnapshots = clientRequestTarget.request().get(VersionedFlowSnapshotMetadata[].class);
    assertNotNull(versionedFlowSnapshots);
    assertEquals(1, versionedFlowSnapshots.length);
    assertFlowSnapshotMetadataEqual(createdFlowSnapshot.getSnapshotMetadata(), versionedFlowSnapshots[0], true);
    // And when the link URI is queried, then the newly created flow snapshot is returned
    final VersionedFlowSnapshot flowSnapshotByLink = client.target(createURL(versionedFlowSnapshots[0].getLink().getUri().toString())).request().get(VersionedFlowSnapshot.class);
    assertFlowSnapshotsEqual(createdFlowSnapshot, flowSnapshotByLink, true);
    assertNotNull(flowSnapshotByLink.getFlow());
    assertNotNull(flowSnapshotByLink.getBucket());
    // And when the bucket is queried by .../versions/{v}, then the newly created flow snapshot is returned
    final VersionedFlowSnapshot flowSnapshotByVersionNumber = clientRequestTarget.path("/1").request().get(VersionedFlowSnapshot.class);
    assertFlowSnapshotsEqual(createdFlowSnapshot, flowSnapshotByVersionNumber, true);
    assertNotNull(flowSnapshotByVersionNumber.getFlow());
    assertNotNull(flowSnapshotByVersionNumber.getBucket());
    // And when the latest URI is queried, then the newly created flow snapshot is returned
    final VersionedFlowSnapshot flowSnapshotByLatest = clientRequestTarget.path("/latest").request().get(VersionedFlowSnapshot.class);
    assertFlowSnapshotsEqual(createdFlowSnapshot, flowSnapshotByLatest, true);
    assertNotNull(flowSnapshotByLatest.getFlow());
    assertNotNull(flowSnapshotByLatest.getBucket());
}

Example 20 with VersionedProcessGroup

use of org.apache.nifi.registry.flow.VersionedProcessGroup in project nifi-registry by apache.

the class UnsecuredNiFiRegistryClientIT method testNiFiRegistryClient.

@Test
public void testNiFiRegistryClient() throws IOException, NiFiRegistryException {
    // ---------------------- TEST BUCKETS --------------------------//
    final BucketClient bucketClient = client.getBucketClient();
    // create buckets
    final int numBuckets = 10;
    final List<Bucket> createdBuckets = new ArrayList<>();
    for (int i = 0; i < numBuckets; i++) {
        final Bucket createdBucket = createBucket(bucketClient, i);
        LOGGER.info("Created bucket # " + i + " with id " + createdBucket.getIdentifier());
        createdBuckets.add(createdBucket);
    }
    // get each bucket
    for (final Bucket bucket : createdBuckets) {
        final Bucket retrievedBucket = bucketClient.get(bucket.getIdentifier());
        Assert.assertNotNull(retrievedBucket);
        LOGGER.info("Retrieved bucket " + retrievedBucket.getIdentifier());
    }
    // final Bucket nonExistentBucket = bucketClient.get("does-not-exist");
    // Assert.assertNull(nonExistentBucket);
    // get bucket fields
    final Fields bucketFields = bucketClient.getFields();
    Assert.assertNotNull(bucketFields);
    LOGGER.info("Retrieved bucket fields, size = " + bucketFields.getFields().size());
    Assert.assertTrue(bucketFields.getFields().size() > 0);
    // get all buckets
    final List<Bucket> allBuckets = bucketClient.getAll();
    LOGGER.info("Retrieved buckets, size = " + allBuckets.size());
    Assert.assertEquals(numBuckets, allBuckets.size());
    allBuckets.stream().forEach(b -> System.out.println("Retrieve bucket " + b.getIdentifier()));
    // update each bucket
    for (final Bucket bucket : createdBuckets) {
        final Bucket bucketUpdate = new Bucket();
        bucketUpdate.setIdentifier(bucket.getIdentifier());
        bucketUpdate.setDescription(bucket.getDescription() + " UPDATE");
        final Bucket updatedBucket = bucketClient.update(bucketUpdate);
        Assert.assertNotNull(updatedBucket);
        LOGGER.info("Updated bucket " + updatedBucket.getIdentifier());
    }
    // ---------------------- TEST FLOWS --------------------------//
    final FlowClient flowClient = client.getFlowClient();
    // create flows
    final Bucket flowsBucket = createdBuckets.get(0);
    final VersionedFlow flow1 = createFlow(flowClient, flowsBucket, 1);
    LOGGER.info("Created flow # 1 with id " + flow1.getIdentifier());
    final VersionedFlow flow2 = createFlow(flowClient, flowsBucket, 2);
    LOGGER.info("Created flow # 2 with id " + flow2.getIdentifier());
    // get flow
    final VersionedFlow retrievedFlow1 = flowClient.get(flowsBucket.getIdentifier(), flow1.getIdentifier());
    Assert.assertNotNull(retrievedFlow1);
    LOGGER.info("Retrieved flow # 1 with id " + retrievedFlow1.getIdentifier());
    final VersionedFlow retrievedFlow2 = flowClient.get(flowsBucket.getIdentifier(), flow2.getIdentifier());
    Assert.assertNotNull(retrievedFlow2);
    LOGGER.info("Retrieved flow # 2 with id " + retrievedFlow2.getIdentifier());
    // update flows
    final VersionedFlow flow1Update = new VersionedFlow();
    flow1Update.setIdentifier(flow1.getIdentifier());
    flow1Update.setName(flow1.getName() + " UPDATED");
    final VersionedFlow updatedFlow1 = flowClient.update(flowsBucket.getIdentifier(), flow1Update);
    Assert.assertNotNull(updatedFlow1);
    LOGGER.info("Updated flow # 1 with id " + updatedFlow1.getIdentifier());
    // get flow fields
    final Fields flowFields = flowClient.getFields();
    Assert.assertNotNull(flowFields);
    LOGGER.info("Retrieved flow fields, size = " + flowFields.getFields().size());
    Assert.assertTrue(flowFields.getFields().size() > 0);
    // get flows in bucket
    final List<VersionedFlow> flowsInBucket = flowClient.getByBucket(flowsBucket.getIdentifier());
    Assert.assertNotNull(flowsInBucket);
    Assert.assertEquals(2, flowsInBucket.size());
    flowsInBucket.stream().forEach(f -> LOGGER.info("Flow in bucket, flow id " + f.getIdentifier()));
    // ---------------------- TEST SNAPSHOTS --------------------------//
    final FlowSnapshotClient snapshotClient = client.getFlowSnapshotClient();
    // create snapshots
    final VersionedFlow snapshotFlow = flow1;
    final VersionedFlowSnapshot snapshot1 = createSnapshot(snapshotClient, snapshotFlow, 1);
    LOGGER.info("Created snapshot # 1 with version " + snapshot1.getSnapshotMetadata().getVersion());
    final VersionedFlowSnapshot snapshot2 = createSnapshot(snapshotClient, snapshotFlow, 2);
    LOGGER.info("Created snapshot # 2 with version " + snapshot2.getSnapshotMetadata().getVersion());
    // get snapshot
    final VersionedFlowSnapshot retrievedSnapshot1 = snapshotClient.get(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier(), 1);
    Assert.assertNotNull(retrievedSnapshot1);
    Assert.assertFalse(retrievedSnapshot1.isLatest());
    LOGGER.info("Retrieved snapshot # 1 with version " + retrievedSnapshot1.getSnapshotMetadata().getVersion());
    final VersionedFlowSnapshot retrievedSnapshot2 = snapshotClient.get(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier(), 2);
    Assert.assertNotNull(retrievedSnapshot2);
    Assert.assertTrue(retrievedSnapshot2.isLatest());
    LOGGER.info("Retrieved snapshot # 2 with version " + retrievedSnapshot2.getSnapshotMetadata().getVersion());
    // get latest
    final VersionedFlowSnapshot retrievedSnapshotLatest = snapshotClient.getLatest(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier());
    Assert.assertNotNull(retrievedSnapshotLatest);
    Assert.assertEquals(snapshot2.getSnapshotMetadata().getVersion(), retrievedSnapshotLatest.getSnapshotMetadata().getVersion());
    Assert.assertTrue(retrievedSnapshotLatest.isLatest());
    LOGGER.info("Retrieved latest snapshot with version " + retrievedSnapshotLatest.getSnapshotMetadata().getVersion());
    // get metadata
    final List<VersionedFlowSnapshotMetadata> retrievedMetadata = snapshotClient.getSnapshotMetadata(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier());
    Assert.assertNotNull(retrievedMetadata);
    Assert.assertEquals(2, retrievedMetadata.size());
    Assert.assertEquals(2, retrievedMetadata.get(0).getVersion());
    Assert.assertEquals(1, retrievedMetadata.get(1).getVersion());
    retrievedMetadata.stream().forEach(s -> LOGGER.info("Retrieved snapshot metadata " + s.getVersion()));
    // get latest metadata
    final VersionedFlowSnapshotMetadata latestMetadata = snapshotClient.getLatestMetadata(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier());
    Assert.assertNotNull(latestMetadata);
    Assert.assertEquals(2, latestMetadata.getVersion());
    // get latest metadata that doesn't exist
    try {
        snapshotClient.getLatestMetadata(snapshotFlow.getBucketIdentifier(), "DOES-NOT-EXIST");
        Assert.fail("Should have thrown exception");
    } catch (NiFiRegistryException nfe) {
        Assert.assertEquals("Error retrieving latest snapshot metadata: The specified flow ID does not exist in this bucket.", nfe.getMessage());
    }
    // ---------------------- TEST ITEMS --------------------------//
    final ItemsClient itemsClient = client.getItemsClient();
    // get fields
    final Fields itemFields = itemsClient.getFields();
    Assert.assertNotNull(itemFields.getFields());
    Assert.assertTrue(itemFields.getFields().size() > 0);
    // get all items
    final List<BucketItem> allItems = itemsClient.getAll();
    Assert.assertEquals(2, allItems.size());
    allItems.stream().forEach(i -> Assert.assertNotNull(i.getBucketName()));
    allItems.stream().forEach(i -> LOGGER.info("All items, item " + i.getIdentifier()));
    // get items for bucket
    final List<BucketItem> bucketItems = itemsClient.getByBucket(flowsBucket.getIdentifier());
    Assert.assertEquals(2, bucketItems.size());
    allItems.stream().forEach(i -> Assert.assertNotNull(i.getBucketName()));
    bucketItems.stream().forEach(i -> LOGGER.info("Items in bucket, item " + i.getIdentifier()));
    // ----------------------- TEST DIFF ---------------------------//
    final VersionedFlowSnapshot snapshot3 = buildSnapshot(snapshotFlow, 3);
    final VersionedProcessGroup newlyAddedPG = new VersionedProcessGroup();
    newlyAddedPG.setIdentifier("new-pg");
    newlyAddedPG.setName("NEW Process Group");
    snapshot3.getFlowContents().getProcessGroups().add(newlyAddedPG);
    snapshotClient.create(snapshot3);
    VersionedFlowDifference diff = flowClient.diff(snapshotFlow.getBucketIdentifier(), snapshotFlow.getIdentifier(), 3, 2);
    Assert.assertNotNull(diff);
    Assert.assertEquals(1, diff.getComponentDifferenceGroups().size());
    // ---------------------- DELETE DATA --------------------------//
    final VersionedFlow deletedFlow1 = flowClient.delete(flowsBucket.getIdentifier(), flow1.getIdentifier());
    Assert.assertNotNull(deletedFlow1);
    LOGGER.info("Deleted flow " + deletedFlow1.getIdentifier());
    final VersionedFlow deletedFlow2 = flowClient.delete(flowsBucket.getIdentifier(), flow2.getIdentifier());
    Assert.assertNotNull(deletedFlow2);
    LOGGER.info("Deleted flow " + deletedFlow2.getIdentifier());
    // delete each bucket
    for (final Bucket bucket : createdBuckets) {
        final Bucket deletedBucket = bucketClient.delete(bucket.getIdentifier());
        Assert.assertNotNull(deletedBucket);
        LOGGER.info("Deleted bucket " + deletedBucket.getIdentifier());
    }
    Assert.assertEquals(0, bucketClient.getAll().size());
    LOGGER.info("!!! SUCCESS !!!");
}