Examples with NiFiProperties org.apache.nifi.util.NiFiProperties used on opensource projects

Example 56 with NiFiProperties

use of org.apache.nifi.util.NiFiProperties in project nifi by apache.

the class NodeClusterCoordinator method afterRequest.

/**
 * Callback that is called after an HTTP Request has been replicated to
 * nodes in the cluster. This allows us to disconnect nodes that did not
 * complete the request, if applicable.
 */
@Override
public void afterRequest(final String uriPath, final String method, final Set<NodeResponse> nodeResponses) {
    // as the cluster coordinator is responsible for performing the actual request replication.
    if (!isActiveClusterCoordinator()) {
        return;
    }
    final boolean mutableRequest = isMutableRequest(method);
    /*
         * Nodes that encountered issues handling the request are marked as
         * disconnected for mutable requests (e.g., post, put, delete). For
         * other requests (e.g., get, head), the nodes remain in their current
         * state even if they had problems handling the request.
         */
    if (mutableRequest) {
        final HttpResponseMapper responseMerger = new StandardHttpResponseMapper(nifiProperties);
        final Set<NodeResponse> problematicNodeResponses = responseMerger.getProblematicNodeResponses(nodeResponses);
        // all nodes failed
        final boolean allNodesFailed = problematicNodeResponses.size() == nodeResponses.size();
        // some nodes had a problematic response because of a missing counter, ensure the are not disconnected
        final boolean someNodesFailedMissingCounter = !problematicNodeResponses.isEmpty() && problematicNodeResponses.size() < nodeResponses.size() && isMissingCounter(problematicNodeResponses, uriPath);
        // ensure nodes stay connected in certain scenarios
        if (allNodesFailed) {
            logger.warn("All nodes failed to process URI {} {}. As a result, no node will be disconnected from cluster", method, uriPath);
            return;
        }
        if (someNodesFailedMissingCounter) {
            return;
        }
        // disconnect problematic nodes
        if (!problematicNodeResponses.isEmpty() && problematicNodeResponses.size() < nodeResponses.size()) {
            final Set<NodeIdentifier> failedNodeIds = problematicNodeResponses.stream().map(response -> response.getNodeId()).collect(Collectors.toSet());
            logger.warn(String.format("The following nodes failed to process URI %s '%s'.  Requesting each node disconnect from cluster.", uriPath, failedNodeIds));
            for (final NodeIdentifier nodeId : failedNodeIds) {
                requestNodeDisconnect(nodeId, DisconnectionCode.FAILED_TO_SERVICE_REQUEST, "Failed to process request " + method + " " + uriPath);
            }
        }
    }
}

Example 57 with NiFiProperties

use of org.apache.nifi.util.NiFiProperties in project nifi by apache.

the class TestPopularVoteFlowElection method getNiFiProperties.

private NiFiProperties getNiFiProperties() {
    final NiFiProperties nifiProperties = mock(NiFiProperties.class);
    when(nifiProperties.getProperty(StringEncryptor.NF_SENSITIVE_PROPS_ALGORITHM)).thenReturn("PBEWITHMD5AND256BITAES-CBC-OPENSSL");
    when(nifiProperties.getProperty(StringEncryptor.NF_SENSITIVE_PROPS_PROVIDER)).thenReturn("BC");
    when(nifiProperties.getProperty(anyString(), anyString())).then(invocation -> invocation.getArgumentAt(1, String.class));
    return nifiProperties;
}

Example 58 with NiFiProperties

use of org.apache.nifi.util.NiFiProperties in project nifi by apache.

the class TestThreadPoolRequestReplicator method testMutableRequestRequiresAllNodesConnected.

@Test
public void testMutableRequestRequiresAllNodesConnected() throws URISyntaxException {
    final ClusterCoordinator coordinator = createClusterCoordinator();
    // build a map of connection state to node ids
    final Map<NodeConnectionState, List<NodeIdentifier>> nodeMap = new HashMap<>();
    final List<NodeIdentifier> connectedNodes = new ArrayList<>();
    connectedNodes.add(new NodeIdentifier("1", "localhost", 8100, "localhost", 8101, "localhost", 8102, 8103, false));
    connectedNodes.add(new NodeIdentifier("2", "localhost", 8200, "localhost", 8201, "localhost", 8202, 8203, false));
    nodeMap.put(NodeConnectionState.CONNECTED, connectedNodes);
    final List<NodeIdentifier> otherState = new ArrayList<>();
    otherState.add(new NodeIdentifier("3", "localhost", 8300, "localhost", 8301, "localhost", 8302, 8303, false));
    nodeMap.put(NodeConnectionState.CONNECTING, otherState);
    when(coordinator.getConnectionStates()).thenReturn(nodeMap);
    final NiFiProperties props = NiFiProperties.createBasicNiFiProperties(null, null);
    final ThreadPoolRequestReplicator replicator = new ThreadPoolRequestReplicator(2, 5, 100, ClientBuilder.newClient(), coordinator, "1 sec", "1 sec", null, null, props) {

        @Override
        public AsyncClusterResponse replicate(Set<NodeIdentifier> nodeIds, String method, URI uri, Object entity, Map<String, String> headers, boolean indicateReplicated, boolean verify) {
            return null;
        }
    };
    try {
        // set the user
        final Authentication authentication = new NiFiAuthenticationToken(new NiFiUserDetails(StandardNiFiUser.ANONYMOUS));
        SecurityContextHolder.getContext().setAuthentication(authentication);
        try {
            replicator.replicate(HttpMethod.POST, new URI("http://localhost:80/processors/1"), new ProcessorEntity(), new HashMap<>());
            Assert.fail("Expected ConnectingNodeMutableRequestException");
        } catch (final ConnectingNodeMutableRequestException e) {
        // expected behavior
        }
        nodeMap.remove(NodeConnectionState.CONNECTING);
        nodeMap.put(NodeConnectionState.DISCONNECTED, otherState);
        try {
            replicator.replicate(HttpMethod.POST, new URI("http://localhost:80/processors/1"), new ProcessorEntity(), new HashMap<>());
            Assert.fail("Expected DisconnectedNodeMutableRequestException");
        } catch (final DisconnectedNodeMutableRequestException e) {
        // expected behavior
        }
        nodeMap.remove(NodeConnectionState.DISCONNECTED);
        nodeMap.put(NodeConnectionState.DISCONNECTING, otherState);
        try {
            replicator.replicate(HttpMethod.POST, new URI("http://localhost:80/processors/1"), new ProcessorEntity(), new HashMap<>());
            Assert.fail("Expected DisconnectedNodeMutableRequestException");
        } catch (final DisconnectedNodeMutableRequestException e) {
        // expected behavior
        }
        // should not throw an Exception because it's a GET
        replicator.replicate(HttpMethod.GET, new URI("http://localhost:80/processors/1"), new MultiValueMap<>(), new HashMap<>());
        // should not throw an Exception because all nodes are now connected
        nodeMap.remove(NodeConnectionState.DISCONNECTING);
        replicator.replicate(HttpMethod.POST, new URI("http://localhost:80/processors/1"), new ProcessorEntity(), new HashMap<>());
    } finally {
        replicator.shutdown();
    }
}

Example 59 with NiFiProperties

use of org.apache.nifi.util.NiFiProperties in project nifi by apache.

the class Cluster method createNode.

public Node createNode() {
    final Map<String, String> addProps = new HashMap<>();
    addProps.put(NiFiProperties.ZOOKEEPER_CONNECT_STRING, getZooKeeperConnectString());
    addProps.put(NiFiProperties.CLUSTER_IS_NODE, "true");
    final NiFiProperties nifiProperties = NiFiProperties.createBasicNiFiProperties("src/test/resources/conf/nifi.properties", addProps);
    final FingerprintFactory fingerprintFactory = new FingerprintFactory(StringEncryptor.createEncryptor(nifiProperties));
    final FlowElection flowElection = new PopularVoteFlowElection(flowElectionTimeoutMillis, TimeUnit.MILLISECONDS, flowElectionMaxNodes, fingerprintFactory);
    final Node node = new Node(nifiProperties, flowElection);
    node.start();
    nodes.add(node);
    return node;
}

Example 60 with NiFiProperties

use of org.apache.nifi.util.NiFiProperties in project nifi by apache.

the class NarUnpackerTest method testUnpackNars.

@Test
public void testUnpackNars() {
    NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties", Collections.EMPTY_MAP);
    assertEquals("./target/NarUnpacker/lib/", properties.getProperty("nifi.nar.library.directory"));
    assertEquals("./target/NarUnpacker/lib2/", properties.getProperty("nifi.nar.library.directory.alt"));
    final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties, SystemBundle.create(properties));
    assertEquals(2, extensionMapping.getAllExtensionNames().size());
    assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.one"));
    assertTrue(extensionMapping.getAllExtensionNames().keySet().contains("org.apache.nifi.processors.dummy.two"));
    final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
    File[] extensionFiles = extensionsWorkingDir.listFiles();
    Set<String> expectedNars = new HashSet<>();
    expectedNars.add("dummy-one.nar-unpacked");
    expectedNars.add("dummy-two.nar-unpacked");
    assertEquals(expectedNars.size(), extensionFiles.length);
    for (File extensionFile : extensionFiles) {
        Assert.assertTrue(expectedNars.contains(extensionFile.getName()));
    }
}