Examples with FlowFileUnpackagerV3 org.apache.nifi.util.FlowFileUnpackagerV3 used on opensource projects

Example 1 with FlowFileUnpackagerV3

use of org.apache.nifi.util.FlowFileUnpackagerV3 in project nifi by apache.

the class UnpackContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    PackageFormat packagingFormat = PackageFormat.getFormat(context.getProperty(PACKAGING_FORMAT).getValue().toLowerCase());
    if (packagingFormat == PackageFormat.AUTO_DETECT_FORMAT) {
        packagingFormat = null;
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No mime.type attribute set for {}; routing to failure", new Object[] { flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
        for (PackageFormat format : PackageFormat.values()) {
            if (mimeType.toLowerCase().equals(format.getMimeType())) {
                packagingFormat = format;
            }
        }
        if (packagingFormat == null) {
            logger.info("Cannot unpack {} because its mime.type attribute is set to '{}', which is not a format that can be unpacked; routing to 'success'", new Object[] { flowFile, mimeType });
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }
    // set the Unpacker to use for this FlowFile.  FlowFileUnpackager objects maintain state and are not reusable.
    final Unpacker unpacker;
    final boolean addFragmentAttrs;
    switch(packagingFormat) {
        case TAR_FORMAT:
        case X_TAR_FORMAT:
            unpacker = tarUnpacker;
            addFragmentAttrs = true;
            break;
        case ZIP_FORMAT:
            unpacker = zipUnpacker;
            addFragmentAttrs = true;
            break;
        case FLOWFILE_STREAM_FORMAT_V2:
            unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV2());
            addFragmentAttrs = false;
            break;
        case FLOWFILE_STREAM_FORMAT_V3:
            unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV3());
            addFragmentAttrs = false;
            break;
        case FLOWFILE_TAR_FORMAT:
            unpacker = new FlowFileStreamUnpacker(new FlowFileUnpackagerV1());
            addFragmentAttrs = false;
            break;
        case AUTO_DETECT_FORMAT:
        default:
            // The format of the unpacker should be known before initialization
            throw new ProcessException(packagingFormat + " is not a valid packaging format");
    }
    final List<FlowFile> unpacked = new ArrayList<>();
    try {
        unpacker.unpack(session, flowFile, unpacked);
        if (unpacked.isEmpty()) {
            logger.error("Unable to unpack {} because it does not appear to have any entries; routing to failure", new Object[] { flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
        if (addFragmentAttrs) {
            finishFragmentAttributes(session, flowFile, unpacked);
        }
        session.transfer(unpacked, REL_SUCCESS);
        final String fragmentId = unpacked.size() > 0 ? unpacked.get(0).getAttribute(FRAGMENT_ID) : null;
        flowFile = FragmentAttributes.copyAttributesToOriginal(session, flowFile, fragmentId, unpacked.size());
        session.transfer(flowFile, REL_ORIGINAL);
        session.getProvenanceReporter().fork(flowFile, unpacked);
        logger.info("Unpacked {} into {} and transferred to success", new Object[] { flowFile, unpacked });
    } catch (final ProcessException | InvalidPathException e) {
        logger.error("Unable to unpack {} due to {}; routing to failure", new Object[] { flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
        session.remove(unpacked);
    }
}

Example 2 with FlowFileUnpackagerV3

use of org.apache.nifi.util.FlowFileUnpackagerV3 in project nifi by apache.

the class ListenHTTPServlet method doPost.

@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response) throws ServletException, IOException {
    final ProcessContext context = processContext;
    ProcessSessionFactory sessionFactory;
    do {
        sessionFactory = sessionFactoryHolder.get();
        if (sessionFactory == null) {
            try {
                Thread.sleep(10);
            } catch (final InterruptedException e) {
            }
        }
    } while (sessionFactory == null);
    final ProcessSession session = sessionFactory.createSession();
    FlowFile flowFile = null;
    String holdUuid = null;
    String foundSubject = null;
    try {
        final long n = filesReceived.getAndIncrement() % FILES_BEFORE_CHECKING_DESTINATION_SPACE;
        if (n == 0 || !spaceAvailable.get()) {
            if (context.getAvailableRelationships().isEmpty()) {
                spaceAvailable.set(false);
                if (logger.isDebugEnabled()) {
                    logger.debug("Received request from " + request.getRemoteHost() + " but no space available; Indicating Service Unavailable");
                }
                response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
                return;
            } else {
                spaceAvailable.set(true);
            }
        }
        response.setHeader("Content-Type", MediaType.TEXT_PLAIN);
        final boolean contentGzipped = Boolean.parseBoolean(request.getHeader(GZIPPED_HEADER));
        final X509Certificate[] certs = (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
        foundSubject = DEFAULT_FOUND_SUBJECT;
        if (certs != null && certs.length > 0) {
            for (final X509Certificate cert : certs) {
                foundSubject = cert.getSubjectDN().getName();
                if (authorizedPattern.matcher(foundSubject).matches()) {
                    break;
                } else {
                    logger.warn("Rejecting transfer attempt from " + foundSubject + " because the DN is not authorized, host=" + request.getRemoteHost());
                    response.sendError(HttpServletResponse.SC_FORBIDDEN, "not allowed based on dn");
                    return;
                }
            }
        }
        final String destinationVersion = request.getHeader(PROTOCOL_VERSION_HEADER);
        Integer protocolVersion = null;
        if (destinationVersion != null) {
            try {
                protocolVersion = Integer.valueOf(destinationVersion);
            } catch (final NumberFormatException e) {
            // Value was invalid. Treat as if the header were missing.
            }
        }
        final boolean destinationIsLegacyNiFi = (protocolVersion == null);
        final boolean createHold = Boolean.parseBoolean(request.getHeader(FLOWFILE_CONFIRMATION_HEADER));
        final String contentType = request.getContentType();
        final InputStream unthrottled = contentGzipped ? new GZIPInputStream(request.getInputStream()) : request.getInputStream();
        final InputStream in = (streamThrottler == null) ? unthrottled : streamThrottler.newThrottledInputStream(unthrottled);
        if (logger.isDebugEnabled()) {
            logger.debug("Received request from " + request.getRemoteHost() + ", createHold=" + createHold + ", content-type=" + contentType + ", gzip=" + contentGzipped);
        }
        final AtomicBoolean hasMoreData = new AtomicBoolean(false);
        final FlowFileUnpackager unpackager;
        if (APPLICATION_FLOW_FILE_V3.equals(contentType)) {
            unpackager = new FlowFileUnpackagerV3();
        } else if (APPLICATION_FLOW_FILE_V2.equals(contentType)) {
            unpackager = new FlowFileUnpackagerV2();
        } else if (APPLICATION_FLOW_FILE_V1.equals(contentType)) {
            unpackager = new FlowFileUnpackagerV1();
        } else {
            unpackager = null;
        }
        final Set<FlowFile> flowFileSet = new HashSet<>();
        do {
            final long startNanos = System.nanoTime();
            final Map<String, String> attributes = new HashMap<>();
            flowFile = session.create();
            flowFile = session.write(flowFile, new OutputStreamCallback() {

                @Override
                public void process(final OutputStream rawOut) throws IOException {
                    try (final BufferedOutputStream bos = new BufferedOutputStream(rawOut, 65536)) {
                        if (unpackager == null) {
                            IOUtils.copy(in, bos);
                            hasMoreData.set(false);
                        } else {
                            attributes.putAll(unpackager.unpackageFlowFile(in, bos));
                            if (destinationIsLegacyNiFi) {
                                if (attributes.containsKey("nf.file.name")) {
                                    // for backward compatibility with old nifi...
                                    attributes.put(CoreAttributes.FILENAME.key(), attributes.remove("nf.file.name"));
                                }
                                if (attributes.containsKey("nf.file.path")) {
                                    attributes.put(CoreAttributes.PATH.key(), attributes.remove("nf.file.path"));
                                }
                            }
                            hasMoreData.set(unpackager.hasMoreData());
                        }
                    }
                }
            });
            final long transferNanos = System.nanoTime() - startNanos;
            final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);
            // put metadata on flowfile
            final String nameVal = request.getHeader(CoreAttributes.FILENAME.key());
            if (StringUtils.isNotBlank(nameVal)) {
                attributes.put(CoreAttributes.FILENAME.key(), nameVal);
            }
            // put arbitrary headers on flow file
            for (Enumeration<String> headerEnum = request.getHeaderNames(); headerEnum.hasMoreElements(); ) {
                String headerName = headerEnum.nextElement();
                if (headerPattern != null && headerPattern.matcher(headerName).matches()) {
                    String headerValue = request.getHeader(headerName);
                    attributes.put(headerName, headerValue);
                }
            }
            String sourceSystemFlowFileIdentifier = attributes.get(CoreAttributes.UUID.key());
            if (sourceSystemFlowFileIdentifier != null) {
                sourceSystemFlowFileIdentifier = "urn:nifi:" + sourceSystemFlowFileIdentifier;
                // If we receveied a UUID, we want to give the FlowFile a new UUID and register the sending system's
                // identifier as the SourceSystemFlowFileIdentifier field in the Provenance RECEIVE event
                attributes.put(CoreAttributes.UUID.key(), UUID.randomUUID().toString());
            }
            flowFile = session.putAllAttributes(flowFile, attributes);
            session.getProvenanceReporter().receive(flowFile, request.getRequestURL().toString(), sourceSystemFlowFileIdentifier, "Remote DN=" + foundSubject, transferMillis);
            flowFile = session.putAttribute(flowFile, "restlistener.remote.source.host", request.getRemoteHost());
            flowFile = session.putAttribute(flowFile, "restlistener.request.uri", request.getRequestURI());
            flowFile = session.putAttribute(flowFile, "restlistener.remote.user.dn", foundSubject);
            flowFileSet.add(flowFile);
            if (holdUuid == null) {
                holdUuid = flowFile.getAttribute(CoreAttributes.UUID.key());
            }
        } while (hasMoreData.get());
        if (createHold) {
            String uuid = (holdUuid == null) ? UUID.randomUUID().toString() : holdUuid;
            if (flowFileMap.containsKey(uuid)) {
                uuid = UUID.randomUUID().toString();
            }
            final FlowFileEntryTimeWrapper wrapper = new FlowFileEntryTimeWrapper(session, flowFileSet, System.currentTimeMillis(), request.getRemoteHost());
            FlowFileEntryTimeWrapper previousWrapper;
            do {
                previousWrapper = flowFileMap.putIfAbsent(uuid, wrapper);
                if (previousWrapper != null) {
                    uuid = UUID.randomUUID().toString();
                }
            } while (previousWrapper != null);
            response.setStatus(HttpServletResponse.SC_SEE_OTHER);
            final String ackUri = "/" + basePath + "/holds/" + uuid;
            response.addHeader(LOCATION_HEADER_NAME, ackUri);
            response.addHeader(LOCATION_URI_INTENT_NAME, LOCATION_URI_INTENT_VALUE);
            response.getOutputStream().write(ackUri.getBytes("UTF-8"));
            if (logger.isDebugEnabled()) {
                logger.debug("Ingested {} from Remote Host: [{}] Port [{}] SubjectDN [{}]; placed hold on these {} files with ID {}", new Object[] { flowFileSet, request.getRemoteHost(), request.getRemotePort(), foundSubject, flowFileSet.size(), uuid });
            }
        } else {
            response.setStatus(this.returnCode);
            logger.info("Received from Remote Host: [{}] Port [{}] SubjectDN [{}]; transferring to 'success' {}", new Object[] { request.getRemoteHost(), request.getRemotePort(), foundSubject, flowFile });
            session.transfer(flowFileSet, ListenHTTP.RELATIONSHIP_SUCCESS);
            session.commit();
        }
    } catch (final Throwable t) {
        session.rollback();
        if (flowFile == null) {
            logger.error("Unable to receive file from Remote Host: [{}] SubjectDN [{}] due to {}", new Object[] { request.getRemoteHost(), foundSubject, t });
        } else {
            logger.error("Unable to receive file {} from Remote Host: [{}] SubjectDN [{}] due to {}", new Object[] { flowFile, request.getRemoteHost(), foundSubject, t });
        }
        response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, t.toString());
    }
}

Example 3 with FlowFileUnpackagerV3

use of org.apache.nifi.util.FlowFileUnpackagerV3 in project nifi by apache.

the class TestPostHTTP method testSendAsFlowFileSecure.

@Test
public void testSendAsFlowFileSecure() throws Exception {
    final Map<String, String> sslProps = new HashMap<>();
    sslProps.put(StandardSSLContextService.KEYSTORE.getName(), "src/test/resources/localhost-ks.jks");
    sslProps.put(StandardSSLContextService.KEYSTORE_PASSWORD.getName(), "localtest");
    sslProps.put(StandardSSLContextService.KEYSTORE_TYPE.getName(), "JKS");
    sslProps.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/localhost-ts.jks");
    sslProps.put(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName(), "localtest");
    sslProps.put(StandardSSLContextService.TRUSTSTORE_TYPE.getName(), "JKS");
    sslProps.put(TestServer.NEED_CLIENT_AUTH, "true");
    setup(sslProps);
    final SSLContextService sslContextService = new StandardSSLContextService();
    runner.addControllerService("ssl-context", sslContextService);
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE, "src/test/resources/localhost-ts.jks");
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_PASSWORD, "localtest");
    runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_TYPE, "JKS");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE, "src/test/resources/localhost-ks.jks");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_PASSWORD, "localtest");
    runner.setProperty(sslContextService, StandardSSLContextService.KEYSTORE_TYPE, "JKS");
    runner.enableControllerService(sslContextService);
    runner.setProperty(PostHTTP.URL, server.getSecureUrl());
    runner.setProperty(PostHTTP.SEND_AS_FLOWFILE, "true");
    runner.setProperty(PostHTTP.SSL_CONTEXT_SERVICE, "ssl-context");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");
    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);
    runner.run(1);
    runner.assertAllFlowFilesTransferred(PostHTTP.REL_SUCCESS);
    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);
    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();
    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));
    assertTrue(unpacker.hasMoreData());
    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();
    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
}

Example 4 with FlowFileUnpackagerV3

use of org.apache.nifi.util.FlowFileUnpackagerV3 in project nifi by apache.

the class TestPostHTTP method checkBatch.

private void checkBatch(TestServer server, CaptureServlet servlet, Set<String> actualContent, int expectedCount) throws Exception {
    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();
    Set<String> actualFFContent = new HashSet<>();
    Set<String> actualPostContent = new HashSet<>();
    runner.assertAllFlowFilesTransferred(PostHTTP.REL_SUCCESS, expectedCount);
    // confirm that all FlowFiles transferred to 'success' have the same URL
    // also accumulate content to verify later
    final List<MockFlowFile> successFlowFiles = runner.getFlowFilesForRelationship(PostHTTP.REL_SUCCESS);
    for (int i = 0; i < expectedCount; i++) {
        MockFlowFile mff = successFlowFiles.get(i);
        mff.assertAttributeEquals("url", server.getUrl());
        String content = new String(mff.toByteArray());
        actualFFContent.add(content);
    }
    // also accumulate content to verify later
    try (ByteArrayInputStream bais = new ByteArrayInputStream(servlet.getLastPost());
        ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
        for (int i = 0; i < expectedCount; i++) {
            Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
            String receivedContent = new String(baos.toByteArray());
            actualPostContent.add(receivedContent);
            assertEquals(server.getUrl(), receivedAttrs.get("url"));
            assertTrue(unpacker.hasMoreData() || i == (expectedCount - 1));
            baos.reset();
        }
    }
    // confirm that the transferred and POSTed content match
    assertEquals(actualFFContent, actualPostContent);
    // accumulate actial content
    actualContent.addAll(actualPostContent);
    runner.clearTransferState();
}

Example 5 with FlowFileUnpackagerV3

use of org.apache.nifi.util.FlowFileUnpackagerV3 in project nifi by apache.

the class TestPostHTTP method testSendAsFlowFile.

@Test
public void testSendAsFlowFile() throws Exception {
    setup(null);
    runner.setProperty(PostHTTP.URL, server.getUrl());
    runner.setProperty(PostHTTP.SEND_AS_FLOWFILE, "true");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("abc", "cba");
    runner.enqueue("Hello".getBytes(), attrs);
    attrs.put("abc", "abc");
    attrs.put("filename", "xyz.txt");
    runner.enqueue("World".getBytes(), attrs);
    runner.run(1);
    runner.assertAllFlowFilesTransferred(PostHTTP.REL_SUCCESS);
    final byte[] lastPost = servlet.getLastPost();
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final ByteArrayInputStream bais = new ByteArrayInputStream(lastPost);
    FlowFileUnpackagerV3 unpacker = new FlowFileUnpackagerV3();
    // unpack first flowfile received
    Map<String, String> receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    byte[] contentReceived = baos.toByteArray();
    assertEquals("Hello", new String(contentReceived));
    assertEquals("cba", receivedAttrs.get("abc"));
    assertTrue(unpacker.hasMoreData());
    baos.reset();
    receivedAttrs = unpacker.unpackageFlowFile(bais, baos);
    contentReceived = baos.toByteArray();
    assertEquals("World", new String(contentReceived));
    assertEquals("abc", receivedAttrs.get("abc"));
    assertEquals("xyz.txt", receivedAttrs.get("filename"));
    Assert.assertNull(receivedAttrs.get("Content-Length"));
}