Examples with ProcessException org.apache.nifi.processor.exception.ProcessException used on opensource projects

Example 51 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class PublishKafkaRecord_1_0 method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final List<FlowFile> flowFiles = session.get(FlowFileFilters.newSizeBasedFilter(1, DataUnit.MB, 500));
    if (flowFiles.isEmpty()) {
        return;
    }
    final PublisherPool pool = getPublisherPool(context);
    if (pool == null) {
        context.yield();
        return;
    }
    final String securityProtocol = context.getProperty(KafkaProcessorUtils.SECURITY_PROTOCOL).getValue();
    final String bootstrapServers = context.getProperty(KafkaProcessorUtils.BOOTSTRAP_SERVERS).evaluateAttributeExpressions().getValue();
    final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
    final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
    final boolean useTransactions = context.getProperty(USE_TRANSACTIONS).asBoolean();
    final long startTime = System.nanoTime();
    try (final PublisherLease lease = pool.obtainPublisher()) {
        if (useTransactions) {
            lease.beginTransaction();
        }
        // Send each FlowFile to Kafka asynchronously.
        final Iterator<FlowFile> itr = flowFiles.iterator();
        while (itr.hasNext()) {
            final FlowFile flowFile = itr.next();
            if (!isScheduled()) {
                // If stopped, re-queue FlowFile instead of sending it
                if (useTransactions) {
                    session.rollback();
                    lease.rollback();
                    return;
                }
                session.transfer(flowFile);
                itr.remove();
                continue;
            }
            final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(flowFile).getValue();
            final String messageKeyField = context.getProperty(MESSAGE_KEY_FIELD).evaluateAttributeExpressions(flowFile).getValue();
            try {
                session.read(flowFile, new InputStreamCallback() {

                    @Override
                    public void process(final InputStream rawIn) throws IOException {
                        try (final InputStream in = new BufferedInputStream(rawIn)) {
                            final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger());
                            final RecordSet recordSet = reader.createRecordSet();
                            final RecordSchema schema = writerFactory.getSchema(flowFile.getAttributes(), recordSet.getSchema());
                            lease.publish(flowFile, recordSet, writerFactory, schema, messageKeyField, topic);
                        } catch (final SchemaNotFoundException | MalformedRecordException e) {
                            throw new ProcessException(e);
                        }
                    }
                });
            } catch (final Exception e) {
                // The FlowFile will be obtained and the error logged below, when calling publishResult.getFailedFlowFiles()
                lease.fail(flowFile, e);
                continue;
            }
        }
        // Complete the send
        final PublishResult publishResult = lease.complete();
        if (publishResult.isFailure()) {
            getLogger().info("Failed to send FlowFile to kafka; transferring to failure");
            session.transfer(flowFiles, REL_FAILURE);
            return;
        }
        // Transfer any successful FlowFiles.
        final long transmissionMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
        for (FlowFile success : flowFiles) {
            final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(success).getValue();
            final int msgCount = publishResult.getSuccessfulMessageCount(success);
            success = session.putAttribute(success, MSG_COUNT, String.valueOf(msgCount));
            session.adjustCounter("Messages Sent", msgCount, true);
            final String transitUri = KafkaProcessorUtils.buildTransitURI(securityProtocol, bootstrapServers, topic);
            session.getProvenanceReporter().send(success, transitUri, "Sent " + msgCount + " messages", transmissionMillis);
            session.transfer(success, REL_SUCCESS);
        }
    }
}

Example 52 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class InferAvroSchema method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final FlowFile original = session.get();
    if (original == null) {
        return;
    }
    try {
        final AtomicReference<String> avroSchema = new AtomicReference<>();
        switch(context.getProperty(INPUT_CONTENT_TYPE).getValue()) {
            case USE_MIME_TYPE:
                avroSchema.set(inferAvroSchemaFromMimeType(original, context, session));
                break;
            case JSON_CONTENT:
                avroSchema.set(inferAvroSchemaFromJSON(original, context, session));
                break;
            case CSV_CONTENT:
                avroSchema.set(inferAvroSchemaFromCSV(original, context, session));
                break;
            default:
                // Shouldn't be possible but just in case
                session.transfer(original, REL_UNSUPPORTED_CONTENT);
                break;
        }
        if (StringUtils.isNotEmpty(avroSchema.get())) {
            String destination = context.getProperty(SCHEMA_DESTINATION).getValue();
            FlowFile avroSchemaFF = null;
            switch(destination) {
                case DESTINATION_ATTRIBUTE:
                    avroSchemaFF = session.putAttribute(session.clone(original), AVRO_SCHEMA_ATTRIBUTE_NAME, avroSchema.get());
                    // Leaves the original CoreAttributes.MIME_TYPE in place.
                    break;
                case DESTINATION_CONTENT:
                    avroSchemaFF = session.write(session.create(), new OutputStreamCallback() {

                        @Override
                        public void process(OutputStream out) throws IOException {
                            out.write(avroSchema.get().getBytes());
                        }
                    });
                    avroSchemaFF = session.putAttribute(avroSchemaFF, CoreAttributes.MIME_TYPE.key(), AVRO_MIME_TYPE);
                    break;
                default:
                    break;
            }
            // Transfer the sessions.
            avroSchemaFF = session.putAttribute(avroSchemaFF, CoreAttributes.FILENAME.key(), (original.getAttribute(CoreAttributes.FILENAME.key()) + AVRO_FILE_EXTENSION));
            session.transfer(avroSchemaFF, REL_SUCCESS);
            session.transfer(original, REL_ORIGINAL);
        } else {
            // If the avroSchema is null then the content type is unknown and therefore unsupported
            session.transfer(original, REL_UNSUPPORTED_CONTENT);
        }
    } catch (Exception ex) {
        getLogger().error("Failed to infer Avro schema for {} due to {}", new Object[] { original, ex });
        session.transfer(original, REL_FAILURE);
    }
}

Example 53 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class ConvertExcelToCSVProcessor method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final String desiredSheetsDelimited = context.getProperty(DESIRED_SHEETS).evaluateAttributeExpressions().getValue();
    final boolean formatValues = context.getProperty(FORMAT_VALUES).asBoolean();
    final CSVFormat csvFormat = CSVUtils.createCSVFormat(context);
    // Switch to 0 based index
    final int firstRow = context.getProperty(ROWS_TO_SKIP).asInteger() - 1;
    final String[] sColumnsToSkip = StringUtils.split(context.getProperty(COLUMNS_TO_SKIP).getValue(), ",");
    final List<Integer> columnsToSkip = new ArrayList<>();
    if (sColumnsToSkip != null && sColumnsToSkip.length > 0) {
        for (String c : sColumnsToSkip) {
            try {
                // Switch to 0 based index
                columnsToSkip.add(Integer.parseInt(c) - 1);
            } catch (NumberFormatException e) {
                throw new ProcessException("Invalid column in Columns to Skip list.", e);
            }
        }
    }
    try {
        session.read(flowFile, new InputStreamCallback() {

            @Override
            public void process(InputStream inputStream) throws IOException {
                try {
                    OPCPackage pkg = OPCPackage.open(inputStream);
                    XSSFReader r = new XSSFReader(pkg);
                    ReadOnlySharedStringsTable sst = new ReadOnlySharedStringsTable(pkg);
                    StylesTable styles = r.getStylesTable();
                    XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) r.getSheetsData();
                    if (desiredSheetsDelimited != null) {
                        String[] desiredSheets = StringUtils.split(desiredSheetsDelimited, DESIRED_SHEETS_DELIMITER);
                        if (desiredSheets != null) {
                            while (iter.hasNext()) {
                                InputStream sheet = iter.next();
                                String sheetName = iter.getSheetName();
                                for (int i = 0; i < desiredSheets.length; i++) {
                                    // If the sheetName is a desired one parse it
                                    if (sheetName.equalsIgnoreCase(desiredSheets[i])) {
                                        ExcelSheetReadConfig readConfig = new ExcelSheetReadConfig(columnsToSkip, firstRow, sheetName, formatValues, sst, styles);
                                        handleExcelSheet(session, flowFile, sheet, readConfig, csvFormat);
                                        break;
                                    }
                                }
                            }
                        } else {
                            getLogger().debug("Excel document was parsed but no sheets with the specified desired names were found.");
                        }
                    } else {
                        // Get all of the sheets in the document.
                        while (iter.hasNext()) {
                            InputStream sheet = iter.next();
                            String sheetName = iter.getSheetName();
                            ExcelSheetReadConfig readConfig = new ExcelSheetReadConfig(columnsToSkip, firstRow, sheetName, formatValues, sst, styles);
                            handleExcelSheet(session, flowFile, sheet, readConfig, csvFormat);
                        }
                    }
                } catch (InvalidFormatException ife) {
                    getLogger().error("Only .xlsx Excel 2007 OOXML files are supported", ife);
                    throw new UnsupportedOperationException("Only .xlsx Excel 2007 OOXML files are supported", ife);
                } catch (OpenXML4JException | SAXException e) {
                    getLogger().error("Error occurred while processing Excel document metadata", e);
                }
            }
        });
        session.transfer(flowFile, ORIGINAL);
    } catch (RuntimeException ex) {
        getLogger().error("Failed to process incoming Excel document. " + ex.getMessage(), ex);
        FlowFile failedFlowFile = session.putAttribute(flowFile, ConvertExcelToCSVProcessor.class.getName() + ".error", ex.getMessage());
        session.transfer(failedFlowFile, FAILURE);
    }
}

Example 54 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class PutFile method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    final Path configuredRootDirPath = Paths.get(context.getProperty(DIRECTORY).evaluateAttributeExpressions(flowFile).getValue());
    final String conflictResponse = context.getProperty(CONFLICT_RESOLUTION).getValue();
    final Integer maxDestinationFiles = context.getProperty(MAX_DESTINATION_FILES).asInteger();
    final ComponentLog logger = getLogger();
    Path tempDotCopyFile = null;
    try {
        final Path rootDirPath = configuredRootDirPath;
        final Path tempCopyFile = rootDirPath.resolve("." + flowFile.getAttribute(CoreAttributes.FILENAME.key()));
        final Path copyFile = rootDirPath.resolve(flowFile.getAttribute(CoreAttributes.FILENAME.key()));
        if (!Files.exists(rootDirPath)) {
            if (context.getProperty(CREATE_DIRS).asBoolean()) {
                Files.createDirectories(rootDirPath);
            } else {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
                logger.error("Penalizing {} and routing to 'failure' because the output directory {} does not exist and Processor is " + "configured not to create missing directories", new Object[] { flowFile, rootDirPath });
                return;
            }
        }
        final Path dotCopyFile = tempCopyFile;
        tempDotCopyFile = dotCopyFile;
        Path finalCopyFile = copyFile;
        final Path finalCopyFileDir = finalCopyFile.getParent();
        if (Files.exists(finalCopyFileDir) && maxDestinationFiles != null) {
            // check if too many files already
            final int numFiles = finalCopyFileDir.toFile().list().length;
            if (numFiles >= maxDestinationFiles) {
                flowFile = session.penalize(flowFile);
                logger.warn("Penalizing {} and routing to 'failure' because the output directory {} has {} files, which exceeds the " + "configured maximum number of files", new Object[] { flowFile, finalCopyFileDir, numFiles });
                session.transfer(flowFile, REL_FAILURE);
                return;
            }
        }
        if (Files.exists(finalCopyFile)) {
            switch(conflictResponse) {
                case REPLACE_RESOLUTION:
                    Files.delete(finalCopyFile);
                    logger.info("Deleted {} as configured in order to replace with the contents of {}", new Object[] { finalCopyFile, flowFile });
                    break;
                case IGNORE_RESOLUTION:
                    session.transfer(flowFile, REL_SUCCESS);
                    logger.info("Transferring {} to success because file with same name already exists", new Object[] { flowFile });
                    return;
                case FAIL_RESOLUTION:
                    flowFile = session.penalize(flowFile);
                    logger.warn("Penalizing {} and routing to failure as configured because file with the same name already exists", new Object[] { flowFile });
                    session.transfer(flowFile, REL_FAILURE);
                    return;
                default:
                    break;
            }
        }
        session.exportTo(flowFile, dotCopyFile, false);
        final String lastModifiedTime = context.getProperty(CHANGE_LAST_MODIFIED_TIME).evaluateAttributeExpressions(flowFile).getValue();
        if (lastModifiedTime != null && !lastModifiedTime.trim().isEmpty()) {
            try {
                final DateFormat formatter = new SimpleDateFormat(FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US);
                final Date fileModifyTime = formatter.parse(lastModifiedTime);
                dotCopyFile.toFile().setLastModified(fileModifyTime.getTime());
            } catch (Exception e) {
                logger.warn("Could not set file lastModifiedTime to {} because {}", new Object[] { lastModifiedTime, e });
            }
        }
        final String permissions = context.getProperty(CHANGE_PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue();
        if (permissions != null && !permissions.trim().isEmpty()) {
            try {
                String perms = stringPermissions(permissions);
                if (!perms.isEmpty()) {
                    Files.setPosixFilePermissions(dotCopyFile, PosixFilePermissions.fromString(perms));
                }
            } catch (Exception e) {
                logger.warn("Could not set file permissions to {} because {}", new Object[] { permissions, e });
            }
        }
        final String owner = context.getProperty(CHANGE_OWNER).evaluateAttributeExpressions(flowFile).getValue();
        if (owner != null && !owner.trim().isEmpty()) {
            try {
                UserPrincipalLookupService lookupService = dotCopyFile.getFileSystem().getUserPrincipalLookupService();
                Files.setOwner(dotCopyFile, lookupService.lookupPrincipalByName(owner));
            } catch (Exception e) {
                logger.warn("Could not set file owner to {} because {}", new Object[] { owner, e });
            }
        }
        final String group = context.getProperty(CHANGE_GROUP).evaluateAttributeExpressions(flowFile).getValue();
        if (group != null && !group.trim().isEmpty()) {
            try {
                UserPrincipalLookupService lookupService = dotCopyFile.getFileSystem().getUserPrincipalLookupService();
                PosixFileAttributeView view = Files.getFileAttributeView(dotCopyFile, PosixFileAttributeView.class);
                view.setGroup(lookupService.lookupPrincipalByGroupName(group));
            } catch (Exception e) {
                logger.warn("Could not set file group to {} because {}", new Object[] { group, e });
            }
        }
        boolean renamed = false;
        for (int i = 0; i < 10; i++) {
            // try rename up to 10 times.
            if (dotCopyFile.toFile().renameTo(finalCopyFile.toFile())) {
                renamed = true;
                // rename was successful
                break;
            }
            // try waiting a few ms to let whatever might cause rename failure to resolve
            Thread.sleep(100L);
        }
        if (!renamed) {
            if (Files.exists(dotCopyFile) && dotCopyFile.toFile().delete()) {
                logger.debug("Deleted dot copy file {}", new Object[] { dotCopyFile });
            }
            throw new ProcessException("Could not rename: " + dotCopyFile);
        } else {
            logger.info("Produced copy of {} at location {}", new Object[] { flowFile, finalCopyFile });
        }
        session.getProvenanceReporter().send(flowFile, finalCopyFile.toFile().toURI().toString(), stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final Throwable t) {
        if (tempDotCopyFile != null) {
            try {
                Files.deleteIfExists(tempDotCopyFile);
            } catch (final Exception e) {
                logger.error("Unable to remove temporary file {} due to {}", new Object[] { tempDotCopyFile, e });
            }
        }
        flowFile = session.penalize(flowFile);
        logger.error("Penalizing {} and transferring to failure due to {}", new Object[] { flowFile, t });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 55 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class PutFileTransfer method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final String hostname = context.getProperty(FileTransfer.HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
    final int maxNumberOfFiles = context.getProperty(FileTransfer.BATCH_SIZE).asInteger();
    int fileCount = 0;
    try (final T transfer = getFileTransfer(context)) {
        do {
            final String rootPath = context.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions(flowFile).getValue();
            final String workingDirPath;
            if (rootPath == null) {
                workingDirPath = null;
            } else {
                File workingDirectory = new File(rootPath);
                if (!workingDirectory.getPath().startsWith("/") && !workingDirectory.getPath().startsWith("\\")) {
                    workingDirectory = new File(transfer.getHomeDirectory(flowFile), workingDirectory.getPath());
                }
                workingDirPath = workingDirectory.getPath().replace("\\", "/");
            }
            final boolean rejectZeroByteFiles = context.getProperty(FileTransfer.REJECT_ZERO_BYTE).asBoolean();
            final ConflictResult conflictResult = identifyAndResolveConflictFile(context.getProperty(FileTransfer.CONFLICT_RESOLUTION).getValue(), transfer, workingDirPath, flowFile, rejectZeroByteFiles, logger);
            if (conflictResult.isTransfer()) {
                final StopWatch stopWatch = new StopWatch();
                stopWatch.start();
                beforePut(flowFile, context, transfer);
                final FlowFile flowFileToTransfer = flowFile;
                final AtomicReference<String> fullPathRef = new AtomicReference<>(null);
                session.read(flowFile, new InputStreamCallback() {

                    @Override
                    public void process(final InputStream in) throws IOException {
                        try (final InputStream bufferedIn = new BufferedInputStream(in)) {
                            if (workingDirPath != null && context.getProperty(SFTPTransfer.CREATE_DIRECTORY).asBoolean()) {
                                transfer.ensureDirectoryExists(flowFileToTransfer, new File(workingDirPath));
                            }
                            fullPathRef.set(transfer.put(flowFileToTransfer, workingDirPath, conflictResult.getFileName(), bufferedIn));
                        }
                    }
                });
                afterPut(flowFile, context, transfer);
                stopWatch.stop();
                final String dataRate = stopWatch.calculateDataRate(flowFile.getSize());
                final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
                logger.info("Successfully transferred {} to {} on remote host {} in {} milliseconds at a rate of {}", new Object[] { flowFile, fullPathRef.get(), hostname, millis, dataRate });
                String fullPathWithSlash = fullPathRef.get();
                if (!fullPathWithSlash.startsWith("/")) {
                    fullPathWithSlash = "/" + fullPathWithSlash;
                }
                final String destinationUri = transfer.getProtocolName() + "://" + hostname + fullPathWithSlash;
                session.getProvenanceReporter().send(flowFile, destinationUri, millis);
            }
            if (conflictResult.isPenalize()) {
                flowFile = session.penalize(flowFile);
            }
            session.transfer(flowFile, conflictResult.getRelationship());
            session.commit();
        } while (isScheduled() && (getRelationships().size() == context.getAvailableRelationships().size()) && (++fileCount < maxNumberOfFiles) && ((flowFile = session.get()) != null));
    } catch (final IOException e) {
        context.yield();
        logger.error("Unable to transfer {} to remote host {} due to {}", new Object[] { flowFile, hostname, e });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
    } catch (final FlowFileAccessException e) {
        context.yield();
        logger.error("Unable to transfer {} to remote host {} due to {}", new Object[] { flowFile, hostname, e.getCause() });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
    } catch (final ProcessException e) {
        context.yield();
        logger.error("Unable to transfer {} to remote host {} due to {}: {}; routing to failure", new Object[] { flowFile, hostname, e, e.getCause() });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
    }
}