Examples with FlowFile org.apache.nifi.flowfile.FlowFile used on opensource projects

Example 21 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project kylo by Teradata.

the class ExportSqoop method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLog();
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        flowFile = session.create();
        logger.info("Created a flow file having uuid: {}", new Object[] { flowFile.getAttribute(CoreAttributes.UUID.key()) });
    } else {
        logger.info("Using an existing flow file having uuid: {}", new Object[] { flowFile.getAttribute(CoreAttributes.UUID.key()) });
    }
    final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).getValue();
    final String kerberosKeyTab = context.getProperty(KERBEROS_KEYTAB).getValue();
    final SqoopConnectionService sqoopConnectionService = context.getProperty(SQOOP_CONNECTION_SERVICE).asControllerService(SqoopConnectionService.class);
    final String sourceHdfsDirectory = context.getProperty(SOURCE_HDFS_DIRECTORY).evaluateAttributeExpressions(flowFile).getValue();
    final String sourceHdfsFileDelimiter = context.getProperty(SOURCE_HDFS_FILE_DELIMITER).evaluateAttributeExpressions(flowFile).getValue();
    final ExportNullInterpretationStrategy sourceNullInterpretationStrategy = ExportNullInterpretationStrategy.valueOf(context.getProperty(SOURCE_NULL_INTERPRETATION_STRATEGY).getValue());
    final String sourceNullCustomStringIdentifier = context.getProperty(SOURCE_NULL_CUSTOM_STRING_IDENTIFIER).evaluateAttributeExpressions(flowFile).getValue();
    final String sourceNullCustomNonStringIdentifier = context.getProperty(SOURCE_NULL_CUSTOM_NON_STRING_IDENTIFIER).evaluateAttributeExpressions(flowFile).getValue();
    final String targetTableName = context.getProperty(TARGET_TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
    final Integer clusterMapTasks = context.getProperty(CLUSTER_MAP_TASKS).evaluateAttributeExpressions(flowFile).asInteger();
    final String systemProperties = context.getProperty(SQOOP_SYSTEM_PROPERTIES).evaluateAttributeExpressions(flowFile).getValue();
    final String additionalArguments = context.getProperty(SQOOP_ADDITIONAL_ARGUMENTS).evaluateAttributeExpressions(flowFile).getValue();
    final String hcatalogDatabase = context.getProperty(HCATALOG_DATABASE).evaluateAttributeExpressions(flowFile).getValue();
    final String hcatalogTable = context.getProperty(HCATALOG_TABLE).evaluateAttributeExpressions(flowFile).getValue();
    final String COMMAND_SHELL = "/bin/bash";
    final String COMMAND_SHELL_FLAGS = "-c";
    final StopWatch stopWatch = new StopWatch(false);
    KerberosConfig kerberosConfig = new KerberosConfig().setLogger(logger).setKerberosPrincipal(kerberosPrincipal).setKerberosKeytab(kerberosKeyTab);
    SqoopExportBuilder sqoopExportBuilder = new SqoopExportBuilder();
    String sqoopExportCommand = sqoopExportBuilder.setLogger(logger).setHcatalogDatabase(hcatalogDatabase).setHcatalogTable(hcatalogTable).setSystemProperties(systemProperties).setAdditionalArguments(additionalArguments).setTargetConnectionString(sqoopConnectionService.getConnectionString()).setTargetUserName(sqoopConnectionService.getUserName()).setPasswordMode(sqoopConnectionService.getPasswordMode()).setTargetPasswordHdfsFile(sqoopConnectionService.getPasswordHdfsFile()).setTargetPasswordPassphrase(sqoopConnectionService.getPasswordPassphrase()).setTargetEnteredPassword(sqoopConnectionService.getEnteredPassword()).setTargetConnectionManager(sqoopConnectionService.getConnectionManager()).setTargetDriver(sqoopConnectionService.getDriver()).setTargetTableName(targetTableName).setSourceHdfsDirectory(sourceHdfsDirectory).setSourceHdfsFileDelimiter(sourceHdfsFileDelimiter).setSourceNullInterpretationStrategy(sourceNullInterpretationStrategy).setSourceNullInterpretationStrategyCustomNullString(sourceNullCustomStringIdentifier).setSourceNullInterpretationStrategyCustomNullNonString(sourceNullCustomNonStringIdentifier).setClusterMapTasks(clusterMapTasks).build();
    List<String> sqoopExportExecutionCommand = new ArrayList<>();
    sqoopExportExecutionCommand.add(COMMAND_SHELL);
    sqoopExportExecutionCommand.add(COMMAND_SHELL_FLAGS);
    sqoopExportExecutionCommand.add(sqoopExportCommand);
    SqoopExportProcessRunner sqoopExportProcessRunner = new SqoopExportProcessRunner(kerberosConfig, sqoopExportExecutionCommand, logger);
    logger.info("Starting execution of Sqoop export command");
    stopWatch.start();
    SqoopProcessResult sqoopExportProcessResult = sqoopExportProcessRunner.execute();
    long jobDurationSeconds = stopWatch.getElapsed(TimeUnit.SECONDS);
    stopWatch.stop();
    logger.info("Finished execution of Sqoop export command");
    int resultExportStatus = sqoopExportProcessResult.getExitValue();
    SqoopUtils sqoopUtils = new SqoopUtils();
    long recordsExportCount = sqoopUtils.getSqoopExportRecordCount(sqoopExportProcessResult, logger);
    String sqoopExportCommandWithCredentialsMasked = sqoopUtils.maskCredentials(sqoopExportCommand, sqoopUtils.getCredentialsToMask());
    flowFile = session.putAttribute(flowFile, "sqoop.export.command.text", sqoopExportCommandWithCredentialsMasked);
    flowFile = session.putAttribute(flowFile, "sqoop.export.result.code", String.valueOf(resultExportStatus));
    flowFile = session.putAttribute(flowFile, "sqoop.export.run.seconds", String.valueOf(jobDurationSeconds));
    flowFile = session.putAttribute(flowFile, "sqoop.export.record.count", String.valueOf(recordsExportCount));
    flowFile = session.putAttribute(flowFile, "sqoop.export.output.table", targetTableName);
    logger.info("Wrote result attributes to flow file");
    if (resultExportStatus == 0) {
        logger.info("Sqoop Export OK [Code {}]", new Object[] { resultExportStatus });
        session.transfer(flowFile, REL_SUCCESS);
    } else {
        logger.info("Sqoop Export FAIL [Code {}]", new Object[] { resultExportStatus });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 22 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestStandardProcessSession method testWriteToOutputStreamWhileReading.

@Test
public void testWriteToOutputStreamWhileReading() throws IOException {
    final ContentClaim claim = contentRepo.create(false);
    try (final OutputStream out = contentRepo.write(claim)) {
        out.write("hello, world".getBytes());
    }
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().contentClaim(claim).addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).size(12L).build();
    flowFileQueue.put(flowFileRecord);
    final FlowFile flowFile = session.get();
    InputStream in = session.read(flowFile);
    try {
        session.write(flowFile);
        Assert.fail("Was able to obtain an OutputStream for a FlowFile while also holding an InputStream for it");
    } catch (final IllegalStateException e) {
    // expected
    } finally {
        in.close();
    }
    // Should now be okay
    try (final OutputStream out = session.write(flowFile)) {
    }
}

Example 23 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestStandardProcessSession method testMigrateWithAppendableStream.

@Test
public void testMigrateWithAppendableStream() throws IOException {
    FlowFile flowFile = session.create();
    flowFile = session.append(flowFile, out -> out.write("1".getBytes()));
    flowFile = session.append(flowFile, out -> out.write("2".getBytes()));
    final StandardProcessSession newSession = new StandardProcessSession(context, () -> false);
    assertTrue(session.isFlowFileKnown(flowFile));
    assertFalse(newSession.isFlowFileKnown(flowFile));
    session.migrate(newSession, Collections.singleton(flowFile));
    assertFalse(session.isFlowFileKnown(flowFile));
    assertTrue(newSession.isFlowFileKnown(flowFile));
    flowFile = newSession.append(flowFile, out -> out.write("3".getBytes()));
    final byte[] buff = new byte[3];
    try (final InputStream in = newSession.read(flowFile)) {
        StreamUtils.fillBuffer(in, buff, true);
        assertEquals(-1, in.read());
    }
    assertTrue(Arrays.equals(new byte[] { '1', '2', '3' }, buff));
    newSession.remove(flowFile);
    newSession.commit();
    session.commit();
}

Example 24 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestStandardProcessSession method testCommitFailureRequeuesFlowFiles.

@Test
public void testCommitFailureRequeuesFlowFiles() {
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).contentClaim(new StandardContentClaim(resourceClaimManager.newResourceClaim("x", "x", "0", true, false), 0L)).contentClaimOffset(0L).size(0L).build();
    flowFileQueue.put(flowFileRecord);
    final FlowFile originalFlowFile = session.get();
    assertTrue(flowFileQueue.isActiveQueueEmpty());
    assertEquals(1, flowFileQueue.getUnacknowledgedQueueSize().getObjectCount());
    final FlowFile modified = session.write(originalFlowFile, new OutputStreamCallback() {

        @Override
        public void process(OutputStream out) throws IOException {
            out.write("Hello".getBytes());
        }
    });
    session.transfer(modified);
    // instruct flowfile repo to throw IOException on update
    flowFileRepo.setFailOnUpdate(true);
    try {
        session.commit();
        Assert.fail("Session commit completed, even though FlowFile Repo threw IOException");
    } catch (final ProcessException pe) {
    // expected behavior because FlowFile Repo will throw IOException
    }
    assertFalse(flowFileQueue.isActiveQueueEmpty());
    assertEquals(1, flowFileQueue.size().getObjectCount());
    assertEquals(0, flowFileQueue.getUnacknowledgedQueueSize().getObjectCount());
}

Example 25 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestStandardProcessSession method testForksNotEmittedIfFilesDeleted.

@Test
public void testForksNotEmittedIfFilesDeleted() throws IOException {
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).build();
    flowFileQueue.put(flowFileRecord);
    final FlowFile orig = session.get();
    final FlowFile newFlowFile = session.create(orig);
    session.remove(newFlowFile);
    session.commit();
    assertEquals(0, provenanceRepo.getEvents(0L, 100000).size());
}