use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.
the class CountersResource method getCounters.
/**
* Retrieves the counters report for this NiFi.
*
* @return A countersEntity.
* @throws InterruptedException if interrupted
*/
@GET
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.APPLICATION_JSON)
// necessary due to a bug in swagger
@Path("")
@ApiOperation(value = "Gets the current counters for this NiFi", notes = NON_GUARANTEED_ENDPOINT, response = CountersEntity.class, authorizations = { @Authorization(value = "Read - /counters") })
@ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") })
public Response getCounters(@ApiParam(value = "Whether or not to include the breakdown per node. Optional, defaults to false", required = false) @QueryParam("nodewise") @DefaultValue(NODEWISE) final Boolean nodewise, @ApiParam(value = "The id of the node where to get the status.", required = false) @QueryParam("clusterNodeId") final String clusterNodeId) throws InterruptedException {
authorizeCounters(RequestAction.READ);
// ensure a valid request
if (Boolean.TRUE.equals(nodewise) && clusterNodeId != null) {
throw new IllegalArgumentException("Nodewise requests cannot be directed at a specific node.");
}
// replicate if necessary
if (isReplicateRequest()) {
// determine where this request should be sent
if (clusterNodeId == null) {
final NodeResponse nodeResponse;
// to the cluster nodes themselves.
if (getReplicationTarget() == ReplicationTarget.CLUSTER_NODES) {
nodeResponse = getRequestReplicator().replicate(HttpMethod.GET, getAbsolutePath(), getRequestParameters(), getHeaders()).awaitMergedResponse();
} else {
nodeResponse = getRequestReplicator().forwardToCoordinator(getClusterCoordinatorNode(), HttpMethod.GET, getAbsolutePath(), getRequestParameters(), getHeaders()).awaitMergedResponse();
}
final CountersEntity entity = (CountersEntity) nodeResponse.getUpdatedEntity();
// ensure there is an updated entity (result of merging) and prune the response as necessary
if (entity != null && !nodewise) {
entity.getCounters().setNodeSnapshots(null);
}
return nodeResponse.getResponse();
} else {
// get the target node and ensure it exists
final NodeIdentifier targetNode = getClusterCoordinator().getNodeIdentifier(clusterNodeId);
if (targetNode == null) {
throw new UnknownNodeException("The specified cluster node does not exist.");
}
return replicate(HttpMethod.GET, targetNode);
}
}
final CountersDTO countersReport = serviceFacade.getCounters();
// create the response entity
final CountersEntity entity = new CountersEntity();
entity.setCounters(countersReport);
// generate the response
return generateOkResponse(entity).build();
}
use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.
the class FlowFileQueueResource method downloadFlowFileContent.
/**
* Gets the content for the specified flowfile in the specified connection.
*
* @param clientId Optional client id. If the client id is not specified, a new one will be generated. This value (whether specified or generated) is included in the response.
* @param connectionId The connection id
* @param flowFileUuid The flowfile uuid
* @param clusterNodeId The cluster node id
* @return The content stream
* @throws InterruptedException if interrupted
*/
@GET
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.WILDCARD)
@Path("{id}/flowfiles/{flowfile-uuid}/content")
@ApiOperation(value = "Gets the content for a FlowFile in a Connection.", response = StreamingOutput.class, authorizations = { @Authorization(value = "Read Source Data - /data/{component-type}/{uuid}") })
@ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") })
public Response downloadFlowFileContent(@ApiParam(value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.", required = false) @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) final ClientIdParameter clientId, @ApiParam(value = "The connection id.", required = true) @PathParam("id") final String connectionId, @ApiParam(value = "The flowfile uuid.", required = true) @PathParam("flowfile-uuid") final String flowFileUuid, @ApiParam(value = "The id of the node where the content exists if clustered.", required = false) @QueryParam("clusterNodeId") final String clusterNodeId) throws InterruptedException {
// replicate if cluster manager
if (isReplicateRequest()) {
// determine where this request should be sent
if (clusterNodeId == null) {
throw new IllegalArgumentException("The id of the node in the cluster is required.");
} else {
// get the target node and ensure it exists
final NodeIdentifier targetNode = getClusterCoordinator().getNodeIdentifier(clusterNodeId);
if (targetNode == null) {
throw new UnknownNodeException("The specified cluster node does not exist.");
}
return replicate(HttpMethod.GET, targetNode);
}
}
// NOTE - deferred authorization so we can consider flowfile attributes in the access decision
// get the uri of the request
final String uri = generateResourceUri("flowfile-queues", connectionId, "flowfiles", flowFileUuid, "content");
// get an input stream to the content
final DownloadableContent content = serviceFacade.getContent(connectionId, flowFileUuid, uri);
// generate a streaming response
final StreamingOutput response = new StreamingOutput() {
@Override
public void write(final OutputStream output) throws IOException, WebApplicationException {
try (InputStream is = content.getContent()) {
// stream the content to the response
StreamUtils.copy(is, output);
// flush the response
output.flush();
}
}
};
// use the appropriate content type
String contentType = content.getType();
if (contentType == null) {
contentType = MediaType.APPLICATION_OCTET_STREAM;
}
return generateOkResponse(response).type(contentType).header("Content-Disposition", String.format("attachment; filename=\"%s\"", content.getFilename())).build();
}
use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.
the class ProvenanceEventResource method submitReplay.
/**
* Creates a new replay request for the content associated with the specified provenance event id.
*
* @param httpServletRequest request
* @param replayRequestEntity The replay request
* @return A provenanceEventEntity
*/
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@Path("replays")
@ApiOperation(value = "Replays content from a provenance event", response = ProvenanceEventEntity.class, authorizations = { @Authorization(value = "Read Component Data - /data/{component-type}/{uuid}"), @Authorization(value = "Write Component Data - /data/{component-type}/{uuid}") })
@ApiResponses(value = { @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") })
public Response submitReplay(@Context final HttpServletRequest httpServletRequest, @ApiParam(value = "The replay request.", required = true) final SubmitReplayRequestEntity replayRequestEntity) {
// ensure the event id is specified
if (replayRequestEntity == null || replayRequestEntity.getEventId() == null) {
throw new IllegalArgumentException("The id of the event must be specified.");
}
// replicate if cluster manager
if (isReplicateRequest()) {
// determine where this request should be sent
if (replayRequestEntity.getClusterNodeId() == null) {
throw new IllegalArgumentException("The id of the node in the cluster is required.");
} else {
return replicate(HttpMethod.POST, replayRequestEntity, replayRequestEntity.getClusterNodeId());
}
}
// handle expects request (usually from the cluster manager)
final String expects = httpServletRequest.getHeader(RequestReplicator.REQUEST_VALIDATION_HTTP_HEADER);
if (expects != null) {
return generateContinueResponse().build();
}
// submit the provenance replay request
final ProvenanceEventDTO event = serviceFacade.submitReplay(replayRequestEntity.getEventId());
event.setClusterNodeId(replayRequestEntity.getClusterNodeId());
// populate the cluster node address
final ClusterCoordinator coordinator = getClusterCoordinator();
if (coordinator != null) {
final NodeIdentifier nodeId = coordinator.getNodeIdentifier(replayRequestEntity.getClusterNodeId());
event.setClusterNodeAddress(nodeId.getApiAddress() + ":" + nodeId.getApiPort());
}
// create a response entity
final ProvenanceEventEntity entity = new ProvenanceEventEntity();
entity.setProvenanceEvent(event);
// generate the response
URI uri = URI.create(generateResourceUri("provenance-events", event.getId()));
return generateCreatedResponse(uri, entity).build();
}
use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.
the class ThreadPoolRequestReplicator method replicate.
/**
* Replicates the request to all nodes in the given set of node identifiers
*
* @param nodeIds the NodeIdentifiers that identify which nodes to send the request to
* @param method the HTTP method to use
* @param uri the URI to send the request to
* @param entity the entity to use
* @param headers the HTTP Headers
* @param performVerification whether or not to verify that all nodes in the cluster are connected and that all nodes can perform request. Ignored if request is not mutable.
* @param response the response to update with the results
* @param executionPhase <code>true</code> if this is the execution phase, <code>false</code> otherwise
* @param monitor a monitor that will be notified when the request completes (successfully or otherwise)
* @return an AsyncClusterResponse that can be used to obtain the result
*/
AsyncClusterResponse replicate(final Set<NodeIdentifier> nodeIds, final String method, final URI uri, final Object entity, final Map<String, String> headers, final boolean performVerification, StandardAsyncClusterResponse response, final boolean executionPhase, final boolean merge, final Object monitor) {
try {
// state validation
Objects.requireNonNull(nodeIds);
Objects.requireNonNull(method);
Objects.requireNonNull(uri);
Objects.requireNonNull(entity);
Objects.requireNonNull(headers);
if (nodeIds.isEmpty()) {
throw new IllegalArgumentException("Cannot replicate request to 0 nodes");
}
// verify all of the nodes exist and are in the proper state
for (final NodeIdentifier nodeId : nodeIds) {
final NodeConnectionStatus status = clusterCoordinator.getConnectionStatus(nodeId);
if (status == null) {
throw new UnknownNodeException("Node " + nodeId + " does not exist in this cluster");
}
if (status.getState() != NodeConnectionState.CONNECTED) {
throw new IllegalClusterStateException("Cannot replicate request to Node " + nodeId + " because the node is not connected");
}
}
logger.debug("Replicating request {} {} with entity {} to {}; response is {}", method, uri, entity, nodeIds, response);
// Update headers to indicate the current revision so that we can
// prevent multiple users changing the flow at the same time
final Map<String, String> updatedHeaders = new HashMap<>(headers);
final String requestId = updatedHeaders.computeIfAbsent(REQUEST_TRANSACTION_ID_HEADER, key -> UUID.randomUUID().toString());
long verifyClusterStateNanos = -1;
if (performVerification) {
final long start = System.nanoTime();
verifyClusterState(method, uri.getPath());
verifyClusterStateNanos = System.nanoTime() - start;
}
int numRequests = responseMap.size();
if (numRequests >= maxConcurrentRequests) {
numRequests = purgeExpiredRequests();
}
if (numRequests >= maxConcurrentRequests) {
final Map<String, Long> countsByUri = responseMap.values().stream().collect(Collectors.groupingBy(StandardAsyncClusterResponse::getURIPath, Collectors.counting()));
logger.error("Cannot replicate request {} {} because there are {} outstanding HTTP Requests already. Request Counts Per URI = {}", method, uri.getPath(), numRequests, countsByUri);
throw new IllegalStateException("There are too many outstanding HTTP requests with a total " + numRequests + " outstanding requests");
}
// create a response object if one was not already passed to us
if (response == null) {
// create the request objects and replicate to all nodes.
// When the request has completed, we need to ensure that we notify the monitor, if there is one.
final CompletionCallback completionCallback = clusterResponse -> {
try {
onCompletedResponse(requestId);
} finally {
if (monitor != null) {
synchronized (monitor) {
monitor.notify();
}
logger.debug("Notified monitor {} because request {} {} has completed", monitor, method, uri);
}
}
};
final Runnable responseConsumedCallback = () -> onResponseConsumed(requestId);
response = new StandardAsyncClusterResponse(requestId, uri, method, nodeIds, responseMapper, completionCallback, responseConsumedCallback, merge);
responseMap.put(requestId, response);
}
if (verifyClusterStateNanos > -1) {
response.addTiming("Verify Cluster State", "All Nodes", verifyClusterStateNanos);
}
logger.debug("For Request ID {}, response object is {}", requestId, response);
// if mutable request, we have to do a two-phase commit where we ask each node to verify
// that the request can take place and then, if all nodes agree that it can, we can actually
// issue the request. This is all handled by calling performVerification, which will replicate
// the 'vote' request to all nodes and then if successful will call back into this method to
// replicate the actual request.
final boolean mutableRequest = isMutableRequest(method, uri.getPath());
if (mutableRequest && performVerification) {
logger.debug("Performing verification (first phase of two-phase commit) for Request ID {}", requestId);
performVerification(nodeIds, method, uri, entity, updatedHeaders, response, merge, monitor);
return response;
} else if (mutableRequest) {
response.setPhase(StandardAsyncClusterResponse.COMMIT_PHASE);
}
// Callback function for generating a NodeHttpRequestCallable that can be used to perform the work
final StandardAsyncClusterResponse finalResponse = response;
NodeRequestCompletionCallback nodeCompletionCallback = nodeResponse -> {
logger.debug("Received response from {} for {} {}", nodeResponse.getNodeId(), method, uri.getPath());
finalResponse.add(nodeResponse);
};
// instruct the node to actually perform the underlying action
if (mutableRequest && executionPhase) {
updatedHeaders.put(REQUEST_EXECUTION_HTTP_HEADER, "true");
}
// replicate the request to all nodes
final Function<NodeIdentifier, NodeHttpRequest> requestFactory = nodeId -> new NodeHttpRequest(nodeId, method, createURI(uri, nodeId), entity, updatedHeaders, nodeCompletionCallback, finalResponse);
submitAsyncRequest(nodeIds, uri.getScheme(), uri.getPath(), requestFactory, updatedHeaders);
return response;
} catch (final Throwable t) {
if (monitor != null) {
synchronized (monitor) {
monitor.notify();
}
logger.debug("Notified monitor {} because request {} {} has failed with Throwable {}", monitor, method, uri, t);
}
if (response != null) {
final RuntimeException failure = (t instanceof RuntimeException) ? (RuntimeException) t : new RuntimeException("Failed to submit Replication Request to background thread", t);
response.setFailure(failure, new NodeIdentifier());
}
throw t;
}
}
use of org.apache.nifi.cluster.protocol.NodeIdentifier in project nifi by apache.
the class ThreadPoolRequestReplicator method performVerification.
private void performVerification(final Set<NodeIdentifier> nodeIds, final String method, final URI uri, final Object entity, final Map<String, String> headers, final StandardAsyncClusterResponse clusterResponse, final boolean merge, final Object monitor) {
logger.debug("Verifying that mutable request {} {} can be made", method, uri.getPath());
final Map<String, String> validationHeaders = new HashMap<>(headers);
validationHeaders.put(REQUEST_VALIDATION_HTTP_HEADER, NODE_CONTINUE);
final long startNanos = System.nanoTime();
final int numNodes = nodeIds.size();
final NodeRequestCompletionCallback completionCallback = new NodeRequestCompletionCallback() {
final Set<NodeResponse> nodeResponses = Collections.synchronizedSet(new HashSet<>());
@Override
public void onCompletion(final NodeResponse nodeResponse) {
// Add the node response to our collection. We later need to know whether or
// not this is the last node response, so we add the response and then check
// the size within a synchronized block to ensure that those two things happen
// atomically. Otherwise, we could have multiple threads checking the sizes of
// the sets at the same time, which could result in multiple threads performing
// the 'all nodes are complete' logic.
final boolean allNodesResponded;
synchronized (nodeResponses) {
nodeResponses.add(nodeResponse);
allNodesResponded = nodeResponses.size() == numNodes;
}
try {
final long nanos = System.nanoTime() - startNanos;
clusterResponse.addTiming("Completed Verification", nodeResponse.getNodeId().toString(), nanos);
// and if good replicate the original request to all of the nodes.
if (allNodesResponded) {
clusterResponse.addTiming("Verification Completed", "All Nodes", nanos);
// Check if we have any requests that do not have a 150-Continue status code.
final long dissentingCount = nodeResponses.stream().filter(p -> p.getStatus() != NODE_CONTINUE_STATUS_CODE).count();
// to all nodes and we are finished.
if (dissentingCount == 0) {
logger.debug("Received verification from all {} nodes that mutable request {} {} can be made", numNodes, method, uri.getPath());
replicate(nodeIds, method, uri, entity, headers, false, clusterResponse, true, merge, monitor);
return;
}
try {
final Map<String, String> cancelLockHeaders = new HashMap<>(headers);
cancelLockHeaders.put(REQUEST_TRANSACTION_CANCELATION_HTTP_HEADER, "true");
final Thread cancelLockThread = new Thread(new Runnable() {
@Override
public void run() {
logger.debug("Found {} dissenting nodes for {} {}; canceling claim request", dissentingCount, method, uri.getPath());
final Function<NodeIdentifier, NodeHttpRequest> requestFactory = nodeId -> new NodeHttpRequest(nodeId, method, createURI(uri, nodeId), entity, cancelLockHeaders, null, clusterResponse);
submitAsyncRequest(nodeIds, uri.getScheme(), uri.getPath(), requestFactory, cancelLockHeaders);
}
});
cancelLockThread.setName("Cancel Flow Locks");
cancelLockThread.start();
// Check that all nodes responded successfully.
for (final NodeResponse response : nodeResponses) {
if (response.getStatus() != NODE_CONTINUE_STATUS_CODE) {
final Response clientResponse = response.getClientResponse();
final String message;
if (clientResponse == null) {
message = "Node " + response.getNodeId() + " is unable to fulfill this request due to: Unexpected Response Code " + response.getStatus();
logger.info("Received a status of {} from {} for request {} {} when performing first stage of two-stage commit. The action will not occur", response.getStatus(), response.getNodeId(), method, uri.getPath());
} else {
final String nodeExplanation = clientResponse.readEntity(String.class);
message = "Node " + response.getNodeId() + " is unable to fulfill this request due to: " + nodeExplanation;
logger.info("Received a status of {} from {} for request {} {} when performing first stage of two-stage commit. " + "The action will not occur. Node explanation: {}", response.getStatus(), response.getNodeId(), method, uri.getPath(), nodeExplanation);
}
// if a node reports forbidden, use that as the response failure
final RuntimeException failure;
if (response.getStatus() == Status.FORBIDDEN.getStatusCode()) {
if (response.hasThrowable()) {
failure = new AccessDeniedException(message, response.getThrowable());
} else {
failure = new AccessDeniedException(message);
}
} else {
if (response.hasThrowable()) {
failure = new IllegalClusterStateException(message, response.getThrowable());
} else {
failure = new IllegalClusterStateException(message);
}
}
clusterResponse.setFailure(failure, response.getNodeId());
}
}
} finally {
if (monitor != null) {
synchronized (monitor) {
monitor.notify();
}
logger.debug("Notified monitor {} because request {} {} has failed due to at least 1 dissenting node", monitor, method, uri);
}
}
}
} catch (final Exception e) {
clusterResponse.add(new NodeResponse(nodeResponse.getNodeId(), method, uri, e));
// to the Cluster Response so that the Cluster Response is complete.
for (final NodeResponse otherResponse : nodeResponses) {
if (otherResponse.getNodeId().equals(nodeResponse.getNodeId())) {
continue;
}
clusterResponse.add(otherResponse);
}
}
}
};
// Callback function for generating a NodeHttpRequestCallable that can be used to perform the work
final Function<NodeIdentifier, NodeHttpRequest> requestFactory = nodeId -> new NodeHttpRequest(nodeId, method, createURI(uri, nodeId), entity, validationHeaders, completionCallback, clusterResponse);
// replicate the 'verification request' to all nodes
submitAsyncRequest(nodeIds, uri.getScheme(), uri.getPath(), requestFactory, validationHeaders);
}
Aggregations