Examples with FlowController org.apache.nifi.controller.FlowController used on opensource projects

Example 1 with FlowController

use of org.apache.nifi.controller.FlowController in project nifi by apache.

the class DtoFactory method createJvmDiagnosticsSnapshotDto.

private JVMDiagnosticsSnapshotDTO createJvmDiagnosticsSnapshotDto(final FlowController flowController) {
    final JVMDiagnosticsSnapshotDTO dto = new JVMDiagnosticsSnapshotDTO();
    final JVMControllerDiagnosticsSnapshotDTO controllerDiagnosticsDto = new JVMControllerDiagnosticsSnapshotDTO();
    final JVMFlowDiagnosticsSnapshotDTO flowDiagnosticsDto = new JVMFlowDiagnosticsSnapshotDTO();
    final JVMSystemDiagnosticsSnapshotDTO systemDiagnosticsDto = new JVMSystemDiagnosticsSnapshotDTO();
    dto.setControllerDiagnostics(controllerDiagnosticsDto);
    dto.setFlowDiagnosticsDto(flowDiagnosticsDto);
    dto.setSystemDiagnosticsDto(systemDiagnosticsDto);
    final SystemDiagnostics systemDiagnostics = flowController.getSystemDiagnostics();
    // flow-related information
    final Set<BundleDTO> bundlesLoaded = ExtensionManager.getAllBundles().stream().map(bundle -> bundle.getBundleDetails().getCoordinate()).sorted((a, b) -> a.getCoordinate().compareTo(b.getCoordinate())).map(this::createBundleDto).collect(Collectors.toCollection(LinkedHashSet::new));
    flowDiagnosticsDto.setActiveEventDrivenThreads(flowController.getActiveEventDrivenThreadCount());
    flowDiagnosticsDto.setActiveTimerDrivenThreads(flowController.getActiveTimerDrivenThreadCount());
    flowDiagnosticsDto.setBundlesLoaded(bundlesLoaded);
    flowDiagnosticsDto.setTimeZone(System.getProperty("user.timezone"));
    flowDiagnosticsDto.setUptime(FormatUtils.formatHoursMinutesSeconds(systemDiagnostics.getUptime(), TimeUnit.MILLISECONDS));
    // controller-related information
    controllerDiagnosticsDto.setClusterCoordinator(flowController.isClusterCoordinator());
    controllerDiagnosticsDto.setPrimaryNode(flowController.isPrimary());
    controllerDiagnosticsDto.setMaxEventDrivenThreads(flowController.getMaxEventDrivenThreadCount());
    controllerDiagnosticsDto.setMaxTimerDrivenThreads(flowController.getMaxTimerDrivenThreadCount());
    // system-related information
    systemDiagnosticsDto.setMaxOpenFileDescriptors(systemDiagnostics.getMaxOpenFileHandles());
    systemDiagnosticsDto.setOpenFileDescriptors(systemDiagnostics.getOpenFileHandles());
    systemDiagnosticsDto.setPhysicalMemoryBytes(systemDiagnostics.getTotalPhysicalMemory());
    systemDiagnosticsDto.setPhysicalMemory(FormatUtils.formatDataSize(systemDiagnostics.getTotalPhysicalMemory()));
    final NumberFormat percentageFormat = NumberFormat.getPercentInstance();
    percentageFormat.setMaximumFractionDigits(2);
    final Set<RepositoryUsageDTO> contentRepoUsage = new HashSet<>();
    for (final Map.Entry<String, StorageUsage> entry : systemDiagnostics.getContentRepositoryStorageUsage().entrySet()) {
        final String repoName = entry.getKey();
        final StorageUsage usage = entry.getValue();
        final RepositoryUsageDTO usageDto = new RepositoryUsageDTO();
        usageDto.setName(repoName);
        usageDto.setFileStoreHash(DigestUtils.sha256Hex(flowController.getContentRepoFileStoreName(repoName)));
        usageDto.setFreeSpace(FormatUtils.formatDataSize(usage.getFreeSpace()));
        usageDto.setFreeSpaceBytes(usage.getFreeSpace());
        usageDto.setTotalSpace(FormatUtils.formatDataSize(usage.getTotalSpace()));
        usageDto.setTotalSpaceBytes(usage.getTotalSpace());
        final double usedPercentage = (usage.getTotalSpace() - usage.getFreeSpace()) / (double) usage.getTotalSpace();
        final String utilization = percentageFormat.format(usedPercentage);
        usageDto.setUtilization(utilization);
        contentRepoUsage.add(usageDto);
    }
    final Set<RepositoryUsageDTO> provRepoUsage = new HashSet<>();
    for (final Map.Entry<String, StorageUsage> entry : systemDiagnostics.getProvenanceRepositoryStorageUsage().entrySet()) {
        final String repoName = entry.getKey();
        final StorageUsage usage = entry.getValue();
        final RepositoryUsageDTO usageDto = new RepositoryUsageDTO();
        usageDto.setName(repoName);
        usageDto.setFileStoreHash(DigestUtils.sha256Hex(flowController.getProvenanceRepoFileStoreName(repoName)));
        usageDto.setFreeSpace(FormatUtils.formatDataSize(usage.getFreeSpace()));
        usageDto.setFreeSpaceBytes(usage.getFreeSpace());
        usageDto.setTotalSpace(FormatUtils.formatDataSize(usage.getTotalSpace()));
        usageDto.setTotalSpaceBytes(usage.getTotalSpace());
        final double usedPercentage = (usage.getTotalSpace() - usage.getFreeSpace()) / (double) usage.getTotalSpace();
        final String utilization = percentageFormat.format(usedPercentage);
        usageDto.setUtilization(utilization);
        provRepoUsage.add(usageDto);
    }
    final RepositoryUsageDTO flowFileRepoUsage = new RepositoryUsageDTO();
    for (final Map.Entry<String, StorageUsage> entry : systemDiagnostics.getProvenanceRepositoryStorageUsage().entrySet()) {
        final String repoName = entry.getKey();
        final StorageUsage usage = entry.getValue();
        flowFileRepoUsage.setName(repoName);
        flowFileRepoUsage.setFileStoreHash(DigestUtils.sha256Hex(flowController.getFlowRepoFileStoreName()));
        flowFileRepoUsage.setFreeSpace(FormatUtils.formatDataSize(usage.getFreeSpace()));
        flowFileRepoUsage.setFreeSpaceBytes(usage.getFreeSpace());
        flowFileRepoUsage.setTotalSpace(FormatUtils.formatDataSize(usage.getTotalSpace()));
        flowFileRepoUsage.setTotalSpaceBytes(usage.getTotalSpace());
        final double usedPercentage = (usage.getTotalSpace() - usage.getFreeSpace()) / (double) usage.getTotalSpace();
        final String utilization = percentageFormat.format(usedPercentage);
        flowFileRepoUsage.setUtilization(utilization);
    }
    systemDiagnosticsDto.setContentRepositoryStorageUsage(contentRepoUsage);
    systemDiagnosticsDto.setCpuCores(systemDiagnostics.getAvailableProcessors());
    systemDiagnosticsDto.setCpuLoadAverage(systemDiagnostics.getProcessorLoadAverage());
    systemDiagnosticsDto.setFlowFileRepositoryStorageUsage(flowFileRepoUsage);
    systemDiagnosticsDto.setMaxHeapBytes(systemDiagnostics.getMaxHeap());
    systemDiagnosticsDto.setMaxHeap(FormatUtils.formatDataSize(systemDiagnostics.getMaxHeap()));
    systemDiagnosticsDto.setProvenanceRepositoryStorageUsage(provRepoUsage);
    // Create the Garbage Collection History info
    final GarbageCollectionHistory gcHistory = flowController.getGarbageCollectionHistory();
    final List<GarbageCollectionDiagnosticsDTO> gcDiagnostics = new ArrayList<>();
    for (final String memoryManager : gcHistory.getMemoryManagerNames()) {
        final List<GarbageCollectionStatus> statuses = gcHistory.getGarbageCollectionStatuses(memoryManager);
        final List<GCDiagnosticsSnapshotDTO> gcSnapshots = new ArrayList<>();
        for (final GarbageCollectionStatus status : statuses) {
            final GCDiagnosticsSnapshotDTO snapshotDto = new GCDiagnosticsSnapshotDTO();
            snapshotDto.setTimestamp(status.getTimestamp());
            snapshotDto.setCollectionCount(status.getCollectionCount());
            snapshotDto.setCollectionMillis(status.getCollectionMillis());
            gcSnapshots.add(snapshotDto);
        }
        final GarbageCollectionDiagnosticsDTO gcDto = new GarbageCollectionDiagnosticsDTO();
        gcDto.setMemoryManagerName(memoryManager);
        gcDto.setSnapshots(gcSnapshots);
        gcDiagnostics.add(gcDto);
    }
    systemDiagnosticsDto.setGarbageCollectionDiagnostics(gcDiagnostics);
    return dto;
}

Example 2 with FlowController

use of org.apache.nifi.controller.FlowController in project nifi by apache.

the class TestStandardControllerServiceProvider method testOrderingOfServices.

@Test
public void testOrderingOfServices() {
    final ProcessGroup procGroup = new MockProcessGroup(controller);
    final FlowController controller = Mockito.mock(FlowController.class);
    Mockito.when(controller.getGroup(Mockito.anyString())).thenReturn(procGroup);
    final StandardControllerServiceProvider provider = new StandardControllerServiceProvider(controller, null, null, stateManagerProvider, variableRegistry, niFiProperties);
    final ControllerServiceNode serviceNode1 = provider.createControllerService(ServiceA.class.getName(), "1", systemBundle.getBundleDetails().getCoordinate(), null, false);
    final ControllerServiceNode serviceNode2 = provider.createControllerService(ServiceB.class.getName(), "2", systemBundle.getBundleDetails().getCoordinate(), null, false);
    setProperty(serviceNode1, ServiceA.OTHER_SERVICE.getName(), "2");
    final Map<String, ControllerServiceNode> nodeMap = new LinkedHashMap<>();
    nodeMap.put("1", serviceNode1);
    nodeMap.put("2", serviceNode2);
    List<List<ControllerServiceNode>> branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(2, branches.size());
    List<ControllerServiceNode> ordered = branches.get(0);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    assertTrue(ordered.get(1) == serviceNode1);
    assertEquals(1, branches.get(1).size());
    assertTrue(branches.get(1).get(0) == serviceNode2);
    nodeMap.clear();
    nodeMap.put("2", serviceNode2);
    nodeMap.put("1", serviceNode1);
    branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(2, branches.size());
    ordered = branches.get(1);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    assertTrue(ordered.get(1) == serviceNode1);
    assertEquals(1, branches.get(0).size());
    assertTrue(branches.get(0).get(0) == serviceNode2);
    // add circular dependency on self.
    nodeMap.clear();
    setProperty(serviceNode1, ServiceA.OTHER_SERVICE_2.getName(), "1");
    nodeMap.put("1", serviceNode1);
    nodeMap.put("2", serviceNode2);
    branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(2, branches.size());
    ordered = branches.get(0);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    assertTrue(ordered.get(1) == serviceNode1);
    nodeMap.clear();
    nodeMap.put("2", serviceNode2);
    nodeMap.put("1", serviceNode1);
    branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(2, branches.size());
    ordered = branches.get(1);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    assertTrue(ordered.get(1) == serviceNode1);
    // add circular dependency once removed. In this case, we won't actually be able to enable these because of the
    // circular dependency because they will never be valid because they will always depend on a disabled service.
    // But we want to ensure that the method returns successfully without throwing a StackOverflowException or anything
    // like that.
    nodeMap.clear();
    final ControllerServiceNode serviceNode3 = provider.createControllerService(ServiceA.class.getName(), "3", systemBundle.getBundleDetails().getCoordinate(), null, false);
    setProperty(serviceNode1, ServiceA.OTHER_SERVICE.getName(), "3");
    setProperty(serviceNode3, ServiceA.OTHER_SERVICE.getName(), "1");
    nodeMap.put("1", serviceNode1);
    nodeMap.put("3", serviceNode3);
    branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(2, branches.size());
    ordered = branches.get(0);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode3);
    assertTrue(ordered.get(1) == serviceNode1);
    nodeMap.clear();
    nodeMap.put("3", serviceNode3);
    nodeMap.put("1", serviceNode1);
    branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(2, branches.size());
    ordered = branches.get(1);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode3);
    assertTrue(ordered.get(1) == serviceNode1);
    // Add multiple completely disparate branches.
    nodeMap.clear();
    setProperty(serviceNode1, ServiceA.OTHER_SERVICE.getName(), "2");
    final ControllerServiceNode serviceNode4 = provider.createControllerService(ServiceB.class.getName(), "4", systemBundle.getBundleDetails().getCoordinate(), null, false);
    final ControllerServiceNode serviceNode5 = provider.createControllerService(ServiceB.class.getName(), "5", systemBundle.getBundleDetails().getCoordinate(), null, false);
    setProperty(serviceNode3, ServiceA.OTHER_SERVICE.getName(), "4");
    nodeMap.put("1", serviceNode1);
    nodeMap.put("2", serviceNode2);
    nodeMap.put("3", serviceNode3);
    nodeMap.put("4", serviceNode4);
    nodeMap.put("5", serviceNode5);
    branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(5, branches.size());
    ordered = branches.get(0);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    assertTrue(ordered.get(1) == serviceNode1);
    assertEquals(1, branches.get(1).size());
    assertTrue(branches.get(1).get(0) == serviceNode2);
    ordered = branches.get(2);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode4);
    assertTrue(ordered.get(1) == serviceNode3);
    assertEquals(1, branches.get(3).size());
    assertTrue(branches.get(3).get(0) == serviceNode4);
    assertEquals(1, branches.get(4).size());
    assertTrue(branches.get(4).get(0) == serviceNode5);
    // create 2 branches both dependent on the same service
    nodeMap.clear();
    setProperty(serviceNode1, ServiceA.OTHER_SERVICE.getName(), "2");
    setProperty(serviceNode3, ServiceA.OTHER_SERVICE.getName(), "2");
    nodeMap.put("1", serviceNode1);
    nodeMap.put("2", serviceNode2);
    nodeMap.put("3", serviceNode3);
    branches = StandardControllerServiceProvider.determineEnablingOrder(nodeMap);
    assertEquals(3, branches.size());
    ordered = branches.get(0);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    assertTrue(ordered.get(1) == serviceNode1);
    ordered = branches.get(1);
    assertEquals(1, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    ordered = branches.get(2);
    assertEquals(2, ordered.size());
    assertTrue(ordered.get(0) == serviceNode2);
    assertTrue(ordered.get(1) == serviceNode3);
}

Example 3 with FlowController

use of org.apache.nifi.controller.FlowController in project nifi by apache.

the class TestStandardRemoteProcessGroupDAO method testVerifyUpdateInputPort.

@Test
public void testVerifyUpdateInputPort() {
    final StandardRemoteProcessGroupDAO dao = new StandardRemoteProcessGroupDAO();
    final String remoteProcessGroupId = "remote-process-group-id";
    final String remoteProcessGroupInputPortId = "remote-process-group-input-port-id";
    final FlowController flowController = mock(FlowController.class);
    final ProcessGroup processGroup = mock(ProcessGroup.class);
    final RemoteProcessGroup remoteProcessGroup = mock(RemoteProcessGroup.class);
    final RemoteGroupPort remoteGroupPort = mock(RemoteGroupPort.class);
    dao.setFlowController(flowController);
    when(flowController.getGroup(any())).thenReturn(processGroup);
    when(processGroup.findRemoteProcessGroup(eq(remoteProcessGroupId))).thenReturn(remoteProcessGroup);
    when(remoteProcessGroup.getInputPort(remoteProcessGroupInputPortId)).thenReturn(remoteGroupPort);
    when(remoteGroupPort.getName()).thenReturn("remote-group-port");
    final RemoteProcessGroupPortDTO dto = new RemoteProcessGroupPortDTO();
    dto.setGroupId(remoteProcessGroupId);
    dto.setId(remoteProcessGroupInputPortId);
    dto.setTargetId(remoteProcessGroupInputPortId);
    final BatchSettingsDTO batchSettings = new BatchSettingsDTO();
    dto.setBatchSettings(batchSettings);
    // Empty input values should pass validation.
    dao.verifyUpdateInputPort(remoteProcessGroupId, dto);
    // Concurrent tasks
    dto.setConcurrentlySchedulableTaskCount(0);
    validate(dao, dto, "Concurrent tasks", "positive integer");
    dto.setConcurrentlySchedulableTaskCount(2);
    validate(dao, dto);
    // Batch count
    batchSettings.setCount(-1);
    validate(dao, dto, "Batch count", "positive integer");
    batchSettings.setCount(0);
    validate(dao, dto);
    batchSettings.setCount(1000);
    validate(dao, dto);
    // Batch size
    batchSettings.setSize("AB");
    validate(dao, dto, "Batch size", "Data Size");
    batchSettings.setSize("10 days");
    validate(dao, dto, "Batch size", "Data Size");
    batchSettings.setSize("300MB");
    validate(dao, dto);
    // Batch duration
    batchSettings.setDuration("AB");
    validate(dao, dto, "Batch duration", "Time Unit");
    batchSettings.setDuration("10 KB");
    validate(dao, dto, "Batch duration", "Time Unit");
    batchSettings.setDuration("10 secs");
    validate(dao, dto);
}

Example 4 with FlowController

use of org.apache.nifi.controller.FlowController in project nifi by apache.

the class StandardFlowServiceFactoryBean method getObject.

@Override
public Object getObject() throws Exception {
    if (flowService == null) {
        final FlowController flowController = applicationContext.getBean("flowController", FlowController.class);
        final RevisionManager revisionManager = applicationContext.getBean("revisionManager", RevisionManager.class);
        if (properties.isNode()) {
            final NodeProtocolSenderListener nodeProtocolSenderListener = applicationContext.getBean("nodeProtocolSenderListener", NodeProtocolSenderListener.class);
            final ClusterCoordinator clusterCoordinator = applicationContext.getBean("clusterCoordinator", ClusterCoordinator.class);
            flowService = StandardFlowService.createClusteredInstance(flowController, properties, nodeProtocolSenderListener, clusterCoordinator, encryptor, revisionManager, authorizer);
        } else {
            flowService = StandardFlowService.createStandaloneInstance(flowController, properties, encryptor, revisionManager, authorizer);
        }
    }
    return flowService;
}

Example 5 with FlowController

use of org.apache.nifi.controller.FlowController in project nifi by apache.

the class TestProcessorLifecycle method buildFlowControllerForTest.

private FlowControllerAndSystemBundle buildFlowControllerForTest(final String propKey, final String propValue) throws Exception {
    final Map<String, String> addProps = new HashMap<>();
    addProps.put(NiFiProperties.ADMINISTRATIVE_YIELD_DURATION, "1 sec");
    addProps.put(NiFiProperties.STATE_MANAGEMENT_CONFIG_FILE, "target/test-classes/state-management.xml");
    addProps.put(NiFiProperties.STATE_MANAGEMENT_LOCAL_PROVIDER_ID, "local-provider");
    addProps.put(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS, MockProvenanceRepository.class.getName());
    addProps.put("nifi.remote.input.socket.port", "");
    addProps.put("nifi.remote.input.secure", "");
    if (propKey != null && propValue != null) {
        addProps.put(propKey, propValue);
    }
    final NiFiProperties nifiProperties = NiFiProperties.createBasicNiFiProperties(propsFile, addProps);
    final Bundle systemBundle = SystemBundle.create(nifiProperties);
    ExtensionManager.discoverExtensions(systemBundle, Collections.emptySet());
    final FlowController flowController = FlowController.createStandaloneInstance(mock(FlowFileEventRepository.class), nifiProperties, mock(Authorizer.class), mock(AuditService.class), null, new VolatileBulletinRepository(), new FileBasedVariableRegistry(nifiProperties.getVariableRegistryPropertiesPaths()), mock(FlowRegistryClient.class));
    return new FlowControllerAndSystemBundle(flowController, systemBundle);
}