Examples with TdchConnectionService com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService used on opensource projects

Example 11 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class StandardTdchConnectionService_Set1_Test method testJdbcDriverClass.

@Test
public void testJdbcDriverClass() throws InitializationException {
    ValidationResult validationResult;
    final TestRunner runner = TestRunners.newTestRunner(TestTdchProcessorForTestingTdchConnectionService.class);
    final TdchConnectionService standardTdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, standardTdchConnectionService);
    runner.assertValid(standardTdchConnectionService);
    // JDBC Driver Class
    validationResult = runner.setProperty(standardTdchConnectionService, StandardTdchConnectionService.JDBC_DRIVER_CLASS_NAME, "");
    Assert.assertEquals(StandardTdchConnectionService.JDBC_DRIVER_CLASS_NAME.getDisplayName(), validationResult.getSubject());
    Assert.assertFalse(validationResult.isValid());
    runner.assertNotValid(standardTdchConnectionService);
    validationResult = runner.setProperty(standardTdchConnectionService, StandardTdchConnectionService.JDBC_DRIVER_CLASS_NAME, StandardTdchConnectionService.DEFAULT_JDBC_DRIVER_CLASS);
    Assert.assertEquals(StandardTdchConnectionService.JDBC_DRIVER_CLASS_NAME.getDisplayName(), validationResult.getSubject());
    Assert.assertTrue(validationResult.isValid());
    runner.assertValid(standardTdchConnectionService);
}

Example 12 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class StandardTdchConnectionService_Set1_Test method testUsername.

@Test
public void testUsername() throws InitializationException {
    ValidationResult validationResult;
    final TestRunner runner = TestRunners.newTestRunner(TestTdchProcessorForTestingTdchConnectionService.class);
    final TdchConnectionService standardTdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, standardTdchConnectionService);
    runner.assertValid(standardTdchConnectionService);
    // Username
    validationResult = runner.setProperty(standardTdchConnectionService, StandardTdchConnectionService.USERNAME, "");
    Assert.assertEquals(StandardTdchConnectionService.USERNAME.getDisplayName(), validationResult.getSubject());
    Assert.assertFalse(validationResult.isValid());
    runner.assertNotValid(standardTdchConnectionService);
    validationResult = runner.setProperty(standardTdchConnectionService, StandardTdchConnectionService.USERNAME, StandardTdchConnectionService.DEFAULT_USERNAME);
    Assert.assertEquals(StandardTdchConnectionService.USERNAME.getDisplayName(), validationResult.getSubject());
    Assert.assertTrue(validationResult.isValid());
    runner.assertValid(standardTdchConnectionService);
}

Example 13 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradata method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLog();
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        flowFile = session.create();
        logger.info("[Hive to Teradata Export via TDCH] Created a flow file having uuid: {}", new Object[] { flowFile.getAttribute(CoreAttributes.UUID.key()) });
    } else {
        logger.info("[Hive to Teradata Export via TDCH] Using an existing flow file having uuid: {}", new Object[] { flowFile.getAttribute(CoreAttributes.UUID.key()) });
    }
    final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).getValue();
    final String kerberosKeyTab = context.getProperty(KERBEROS_KEYTAB).getValue();
    final TdchConnectionService tdchConnectionService = context.getProperty(TDCH_CONNECTION_SERVICE).asControllerService(TdchConnectionService.class);
    final String commonExportToolMethod = StringUtils.isEmpty(context.getProperty(EXPORT_TOOL_METHOD).evaluateAttributeExpressions(flowFile).getValue()) ? DEFAULT_EXPORT_TOOL_METHOD : context.getProperty(EXPORT_TOOL_METHOD).evaluateAttributeExpressions(flowFile).getValue();
    final Integer commonNumberOfMappers = StringUtils.isEmpty(context.getProperty(NUMBER_OF_MAPPERS).evaluateAttributeExpressions(flowFile).getValue()) ? Integer.valueOf(DEFAULT_NUMBER_OF_MAPPERS) : context.getProperty(NUMBER_OF_MAPPERS).evaluateAttributeExpressions(flowFile).asInteger();
    final Boolean commonThrottleMappersFlag = StringUtils.isEmpty(context.getProperty(THROTTLE_MAPPERS_FLAG).evaluateAttributeExpressions(flowFile).getValue()) ? Boolean.valueOf(DEFAULT_THROTTLE_MAPPERS_FLAG) : context.getProperty(THROTTLE_MAPPERS_FLAG).evaluateAttributeExpressions(flowFile).asBoolean();
    final Integer commonMinimumMappers = StringUtils.isEmpty(context.getProperty(MINIMUM_MAPPERS).evaluateAttributeExpressions(flowFile).getValue()) ? null : context.getProperty(MINIMUM_MAPPERS).evaluateAttributeExpressions(flowFile).asInteger();
    final String commonSourceDateFormat = context.getProperty(SOURCE_DATE_FORMAT).evaluateAttributeExpressions(flowFile).getValue();
    final String commonSourceTimeFormat = context.getProperty(SOURCE_TIME_FORMAT).evaluateAttributeExpressions(flowFile).getValue();
    final String commonSourceTimestampFormat = context.getProperty(SOURCE_TIMESTAMP_FORMAT).evaluateAttributeExpressions(flowFile).getValue();
    final String commonSourceTimezoneId = context.getProperty(SOURCE_TIMEZONE_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String commonTargetDateFormat = context.getProperty(TARGET_DATE_FORMAT).evaluateAttributeExpressions(flowFile).getValue();
    final String commonTargetTimeFormat = context.getProperty(TARGET_TIME_FORMAT).evaluateAttributeExpressions(flowFile).getValue();
    final String commonTargetTimestampFormat = context.getProperty(TARGET_TIMESTAMP_FORMAT).evaluateAttributeExpressions(flowFile).getValue();
    final String commonTargetTimezoneId = context.getProperty(TARGET_TIMEZONE_ID).evaluateAttributeExpressions(flowFile).getValue();
    final Boolean commonStringTruncateFlag = StringUtils.isEmpty(context.getProperty(STRING_TRUNCATE_FLAG).evaluateAttributeExpressions(flowFile).getValue()) ? Boolean.valueOf(DEFAULT_STRING_TRUNCATE_FLAG) : context.getProperty(STRING_TRUNCATE_FLAG).evaluateAttributeExpressions(flowFile).asBoolean();
    final String commonCharset = context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue();
    final String commonExportToolJobType = StringUtils.isEmpty(context.getProperty(EXPORT_TOOL_JOB_TYPE).evaluateAttributeExpressions(flowFile).getValue()) ? DEFAULT_EXPORT_TOOL_JOB_TYPE : context.getProperty(EXPORT_TOOL_JOB_TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final String commonExportToolFileFormat = StringUtils.isEmpty(context.getProperty(EXPORT_TOOL_FILEFORMAT).evaluateAttributeExpressions(flowFile).getValue()) ? DEFAULT_EXPORT_TOOL_FILEFORMAT : context.getProperty(EXPORT_TOOL_FILEFORMAT).evaluateAttributeExpressions(flowFile).getValue();
    final String hiveConfigurationFileHdfsPath = context.getProperty(HIVE_CONFIGURATION_FILE_HDFS_PATH).evaluateAttributeExpressions(flowFile).getValue();
    final String hiveDatabase = context.getProperty(HIVE_DATABASE).evaluateAttributeExpressions(flowFile).getValue();
    final String hiveTable = context.getProperty(HIVE_TABLE).evaluateAttributeExpressions(flowFile).getValue();
    final String hiveFieldNames = context.getProperty(HIVE_FIELD_NAMES).evaluateAttributeExpressions(flowFile).getValue();
    final String hiveFieldSeparator = context.getProperty(HIVE_FIELD_SEPARATOR).evaluateAttributeExpressions(flowFile).getValue();
    final String hiveLineSeparator = context.getProperty(HIVE_LINE_SEPARATOR).evaluateAttributeExpressions(flowFile).getValue();
    final String teradataDatabaseTable = context.getProperty(TERADATA_DATABASE_TABLE).evaluateAttributeExpressions(flowFile).getValue();
    final String teradataFieldNames = context.getProperty(TERADATA_FIELD_NAMES).evaluateAttributeExpressions(flowFile).getValue();
    final Boolean teradataTruncateTable = StringUtils.isEmpty(context.getProperty(TERADATA_TRUNCATE_TABLE).evaluateAttributeExpressions(flowFile).getValue()) ? Boolean.valueOf(DEFAULT_TERADATA_TRUNCATE_TABLE) : context.getProperty(TERADATA_TRUNCATE_TABLE).evaluateAttributeExpressions(flowFile).asBoolean();
    final Boolean teradataUseXviews = StringUtils.isEmpty(context.getProperty(TERADATA_USE_XVIEWS).evaluateAttributeExpressions(flowFile).getValue()) ? Boolean.valueOf(DEFAULT_TERADATA_USE_XVIEWS) : context.getProperty(TERADATA_USE_XVIEWS).evaluateAttributeExpressions(flowFile).asBoolean();
    final String teradataQueryBand = context.getProperty(TERADATA_QUERY_BAND).evaluateAttributeExpressions(flowFile).getValue();
    final Integer teradataBatchSize = StringUtils.isEmpty(context.getProperty(TERADATA_BATCH_SIZE).evaluateAttributeExpressions(flowFile).getValue()) ? Integer.valueOf(DEFAULT_TERADATA_BATCH_SIZE) : context.getProperty(TERADATA_BATCH_SIZE).evaluateAttributeExpressions(flowFile).asInteger();
    final String teradataStagingDatabase = context.getProperty(TERADATA_STAGING_DATABASE).evaluateAttributeExpressions(flowFile).getValue();
    final String teradataStagingTable = context.getProperty(TERADATA_STAGING_TABLE).evaluateAttributeExpressions(flowFile).getValue();
    final Boolean teradataForceStage = StringUtils.isEmpty(context.getProperty(TERADATA_FORCE_STAGE).evaluateAttributeExpressions(flowFile).getValue()) ? Boolean.valueOf(DEFAULT_TERADATA_FORCE_STAGE) : context.getProperty(TERADATA_FORCE_STAGE).evaluateAttributeExpressions(flowFile).asBoolean();
    final Boolean teradataKeepStagingTable = StringUtils.isEmpty(context.getProperty(TERADATA_KEEP_STAGE_TABLE).evaluateAttributeExpressions(flowFile).getValue()) ? Boolean.valueOf(DEFAULT_TERADATA_KEEP_STAGE) : context.getProperty(TERADATA_KEEP_STAGE_TABLE).evaluateAttributeExpressions(flowFile).asBoolean();
    final String teradataFastLoadErrorDatabase = context.getProperty(TERADATA_FAST_LOAD_ERROR_DATABASE).evaluateAttributeExpressions(flowFile).getValue();
    final String teradataFastLoadErrorTable = context.getProperty(TERADATA_FAST_LOAD_ERROR_TABLE).evaluateAttributeExpressions(flowFile).getValue();
    final StopWatch stopWatch = new StopWatch(false);
    KerberosConfig kerberosConfig = new KerberosConfig().setLogger(logger).setKerberosPrincipal(kerberosPrincipal).setKerberosKeytab(kerberosKeyTab);
    TdchBuilder tdchBuilder = new TdchBuilder();
    String tdchCommand = tdchBuilder.setLogger(logger).setTdchJarEnvironmentVariable(TDCH_JAR_PATH_ENV_VAR_NAME).setTdchLibraryJarsVariable(TDCH_LIB_JARS_ENV_VAR_NAME).setTdchHadoopClassPathVariable(TDCH_HADOOP_CLASSPATH_ENV_VAR_NAME).setTdchOperationType(TdchOperationType.TDCH_EXPORT).setCommonTeradataUrl(tdchConnectionService.getJdbcConnectionUrl(), teradataDatabaseTable, commonCharset).setCommonTeradataClassname(tdchConnectionService.getJdbcDriverClassName()).setCommonTeradataUsername(tdchConnectionService.getUserName()).setCommonTeradataPassword(tdchConnectionService.getPassword()).setCommonExportToolMethod(commonExportToolMethod).setCommonExportToolJobType(commonExportToolJobType).setCommonExportToolFileFormat(commonExportToolFileFormat).setCommonNumMappers(commonNumberOfMappers).setCommonThrottleMappers(commonThrottleMappersFlag).setCommonMinMappers(commonMinimumMappers).setCommonSourceDateFormat(commonSourceDateFormat).setCommonSourceTimeFormat(commonSourceTimeFormat).setCommonSourceTimestampFormat(commonSourceTimestampFormat).setCommonSourceTimezoneId(commonSourceTimezoneId).setCommonTargetDateFormat(commonTargetDateFormat).setCommonTargetTimeFormat(commonTargetTimeFormat).setCommonTargetTimestampFormat(commonTargetTimestampFormat).setCommonTargetTimezoneId(commonTargetTimezoneId).setCommonStringTruncate(commonStringTruncateFlag).setCommonCharset(commonCharset).setSourceHiveConfigurationFileHdfsPath(hiveConfigurationFileHdfsPath).setSourceHiveSourceDatabase(hiveDatabase).setSourceHiveSourceTable(hiveTable).setSourceHiveSourceFieldNames(hiveFieldNames).setSourceHiveFieldSeparator(hiveFieldSeparator).setSourceHiveLineSeparator(hiveLineSeparator).setTargetTeradataDatabaseTable(teradataDatabaseTable).setTargetTeradataTargetFieldNames(teradataFieldNames).setTargetTeradataTruncateTable(teradataTruncateTable).setTargetTeradataUseXviews(teradataUseXviews).setTargetTeradataQueryBand(teradataQueryBand).setTargetTeradataBatchSize(teradataBatchSize).setTargetTeradataStagingDatabase(teradataStagingDatabase).setTargetTeradataStagingTableName(teradataStagingTable).setTargetTeradataForceStage(teradataForceStage).setTargetTeradataKeepStageTable(teradataKeepStagingTable).setTargetTeradataFastLoadErrorTableDatabase(teradataFastLoadErrorDatabase).setTargetTeradataFastLoadErrorTableName(teradataFastLoadErrorTable).build();
    List<String> tdchExecutionCommand = new ArrayList<>();
    tdchExecutionCommand.add(COMMAND_SHELL);
    tdchExecutionCommand.add(COMMAND_SHELL_FLAGS);
    tdchExecutionCommand.add(tdchCommand);
    Map<String, String> tdchEnvironmentVariables = new HashMap<>();
    tdchEnvironmentVariables.put(TDCH_JAR_PATH_ENV_VAR_NAME, tdchConnectionService.getTdchJarPath());
    tdchEnvironmentVariables.put(TDCH_LIB_JARS_ENV_VAR_NAME, tdchConnectionService.getTdchLibraryJarsPath());
    tdchEnvironmentVariables.put(TDCH_HADOOP_CLASSPATH_ENV_VAR_NAME, tdchConnectionService.getTdchHadoopClassPath());
    TdchProcessRunner tdchProcessRunner = new TdchProcessRunner(kerberosConfig, tdchExecutionCommand, logger, TdchOperationType.TDCH_EXPORT, tdchEnvironmentVariables);
    logger.info("Starting execution of TDCH command (Hive to Teradata export)");
    stopWatch.start();
    TdchProcessResult tdchProcessResult = tdchProcessRunner.execute();
    stopWatch.stop();
    logger.info("Finished execution of TDCH command (Hive to Teradata export)");
    int resultStatus = tdchProcessResult.getExitValue();
    TdchUtils tdchUtils = new TdchUtils();
    String tdchCommandWithCredentialsMasked = tdchUtils.maskTdchCredentials(tdchCommand);
    flowFile = session.putAttribute(flowFile, "tdch.export.hive.to.teradata.command", tdchCommandWithCredentialsMasked);
    flowFile = session.putAttribute(flowFile, "tdch.export.hive.to.teradata.kylo.result.code", String.valueOf(resultStatus));
    flowFile = session.putAttribute(flowFile, "tdch.export.hive.to.teradata.input.record.count", String.valueOf(tdchUtils.getExportHiveToTeradataInputRecordsCount(tdchProcessResult, logger)));
    flowFile = session.putAttribute(flowFile, "tdch.export.hive.to.teradata.output.record.count", String.valueOf(tdchUtils.getExportHiveToTeradataOutputRecordsCount(tdchProcessResult, logger)));
    flowFile = session.putAttribute(flowFile, "tdch.export.hive.to.teradata.tdch.exit.code", String.valueOf(tdchUtils.getExportHiveToTeradataJobExitCode(tdchProcessResult, logger)));
    flowFile = session.putAttribute(flowFile, "tdch.export.hive.to.teradata.tdch.time.taken", tdchUtils.getExportHiveToTeradataJobTimeTaken(tdchProcessResult, logger));
    logger.info("Wrote result attributes to flow file");
    if (resultStatus == 0) {
        logger.info("TDCH Hive to Teradata export OK [Code {}]", new Object[] { resultStatus });
        session.transfer(flowFile, REL_SUCCESS);
    } else {
        logger.error("TDCH Hive to Teradata export FAIL [Code {}]", new Object[] { resultStatus });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 14 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testExport_HiveRcFileToTeradataBatchInsert_5_8.

@Test
public void testExport_HiveRcFileToTeradataBatchInsert_5_8() throws InitializationException {
    /*
        From actual run:
        Key: 'tdch.export.hive.to.teradata.command'
	Value: 'hadoop jar $USERLIBTDCH com.teradata.connector.common.tool.ConnectorExportTool -libjars $LIB_JARS -Dtdch.output.teradata.truncate=false -classname "com.teradata.jdbc.TeraDriver" -url "jdbc:teradata://localhost/database=finance" -username "dbc" -password ***** -method "batch.insert" -jobtype "hive" -fileformat "rcfile" -nummappers "2" -throttlemappers "false" -sourcedateformat "yyyy-MM-dd" -sourcetimeformat "HH:mm:ss" -sourcetimestampformat "yyyy-MM-dd HH:mm:ss.SSS" -targetdateformat "yyyy-MM-dd" -targettimeformat "HH:mm:ss" -targettimestampformat "yyyy-MM-dd HH:mm:ss.SSS" -stringtruncate "true" -sourcedatabase "tdch" -sourcetable "example7_hive" -sourcefieldnames "h1,h2" -targettable "finance.example7_td" -targetfieldnames "c1,c2" -usexviews "false" -batchsize "10000" -forcestage "false" -keepstagetable "false" '
        */
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DevTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "${hive_db}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "${hive_table}");
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "${teradata_db}.${teradata_table}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_FIELD_NAMES, "${hive_field_names}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_FIELD_NAMES, "${teradata_field_names}");
    runner.assertValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_JOB_TYPE, "${my.custom.var.export.tool.job.type}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_JOB_TYPE, "${tdch.export.tool.job.type}");
    runner.assertValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_FILEFORMAT, "${my.custom.var.export.tool.file.format}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_FILEFORMAT, "${tdch.export.tool.file.format}");
    runner.assertValid();
    MockFlowFile mockFlowFile = new MockFlowFile(1L);
    Map<String, String> attributes = new HashMap<>();
    attributes.put("tdch.export.tool.job.type", "hive");
    attributes.put("tdch.export.tool.file.format", "rcfile");
    attributes.put("hive_db", "tdch");
    attributes.put("hive_table", "example7_hive");
    attributes.put("teradata_db", "finance");
    attributes.put("teradata_table", "example7_td");
    attributes.put("hive_field_names", "h1,h2");
    attributes.put("teradata_field_names", "c1, c2");
    mockFlowFile.putAttributes(attributes);
    runner.enqueue(mockFlowFile);
    runner.run(1);
    List<MockFlowFile> failedFlowFiles = runner.getFlowFilesForRelationship(TdchExportHiveToTeradata.REL_FAILURE);
    Assert.assertEquals(1, failedFlowFiles.size());
    runner.assertQueueEmpty();
    String expectedCommand = "hadoop jar $USERLIBTDCH com.teradata.connector.common.tool.ConnectorExportTool -libjars $LIB_JARS -Dtdch.output.teradata.truncate=false -classname \"com.teradata.jdbc.TeraDriver\" -url \"jdbc:teradata://localhost/database=finance\" -username \"dbc\" -password ***** -method \"batch.insert\" -jobtype \"hive\" -fileformat \"rcfile\" -nummappers \"2\" -throttlemappers \"false\" -sourcedateformat \"yyyy-MM-dd\" -sourcetimeformat \"HH:mm:ss\" -sourcetimestampformat \"yyyy-MM-dd HH:mm:ss.SSS\" -targetdateformat \"yyyy-MM-dd\" -targettimeformat \"HH:mm:ss\" -targettimestampformat \"yyyy-MM-dd HH:mm:ss.SSS\" -stringtruncate \"true\" -sourcedatabase \"tdch\" -sourcetable \"example7_hive\" -sourcefieldnames \"h1,h2\" -targettable \"finance.example7_td\" -targetfieldnames \"c1,c2\" -usexviews \"false\" -batchsize \"10000\" -forcestage \"false\" -keepstagetable \"false\" ";
    MockFlowFile failedFlowFile = failedFlowFiles.get(0);
    Assert.assertEquals(expectedCommand, failedFlowFile.getAttribute("tdch.export.hive.to.teradata.command"));
}

Example 15 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testExport_HiveRcFileToTeradataBatchInsert_5_8_SetRequiredAndDefaultPropertiesViaExpressionLanguageToValidValues.

@Test
public void testExport_HiveRcFileToTeradataBatchInsert_5_8_SetRequiredAndDefaultPropertiesViaExpressionLanguageToValidValues() throws InitializationException {
    /*
        From actual run:
        Key: 'tdch.export.hive.to.teradata.command'
	Value: 'hadoop jar $USERLIBTDCH com.teradata.connector.common.tool.ConnectorExportTool -libjars $LIB_JARS -Dtdch.output.teradata.truncate=true -classname "com.teradata.jdbc.TeraDriver" -url "jdbc:teradata://localhost/database=finance" -username "dbc" -password ***** -method "batch.insert" -jobtype "hive" -fileformat "rcfile" -nummappers "5" -throttlemappers "true" -sourcedateformat "yy-MM-dd" -sourcetimeformat "HH:mm" -sourcetimestampformat "yyyy-MM-dd HH:mm:ss" -targetdateformat "yy-MM-dd" -targettimeformat "HH:mm" -targettimestampformat "yyyy-MM-dd HH:mm:ss" -stringtruncate "false" -sourcedatabase "tdch" -sourcetable "example7_hive" -sourcefieldnames "h1,h2" -targettable "finance.example7_td" -targetfieldnames "c1,c2" -usexviews "true" -batchsize "500" -forcestage "true" -keepstagetable "true" '
         */
    // This test covers assigning valid values to all properties that are either required or get a default value. It ensures that all of them can be set via expression variables.
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DevTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    // These are required, and support arbitrary expression variable
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "${required_config_hive_db}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "${required_config_hive_table}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "${required_config_teradata_db}.${required_config_teradata_table}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE, "${config_teradata_truncate_table}");
    // These are optional (but get defaults when processor is instantiated), and support arbitrary expression variable
    runner.setProperty(TdchExportHiveToTeradata.HIVE_FIELD_NAMES, "${config_hive_field_names}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_FIELD_NAMES, "${config_teradata_field_names}");
    runner.setProperty(TdchExportHiveToTeradata.NUMBER_OF_MAPPERS, "${config_num_mappers}");
    runner.setProperty(TdchExportHiveToTeradata.THROTTLE_MAPPERS_FLAG, "${config_throttle_mappers_flag}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_SOURCE_DATE_FORMAT, "${config_hive_source_date_format}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_SOURCE_TIME_FORMAT, "${config_hive_source_time_format}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_SOURCE_TIMESTAMP_FORMAT, "${config_hive_source_timestamp_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TARGET_DATE_FORMAT, "${config_teradata_target_date_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TARGET_TIME_FORMAT, "${config_teradata_target_time_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TARGET_TIMESTAMP_FORMAT, "${config_teradata_target_timestamp_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_STRING_TRUNCATE_FLAG, "${config_teradata_string_truncate_flag}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_CHARSET, "${config_teradata_charset}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "${config_teradata_use_xviews}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_BATCH_SIZE, "${config_teradata_batch_size}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_FORCE_STAGE, "${config_teradata_force_stage}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_KEEP_STAGE_TABLE, "${config_teradata_keep_stage_table}");
    runner.assertValid();
    // These need the specific expression language variable since processor checks them. The test verifies this.
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_EXPORT_TOOL_METHOD, "${my.custom.var.export.tool.method}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_EXPORT_TOOL_METHOD, "${tdch.export.tool.method}");
    runner.assertValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_JOB_TYPE, "${my.custom.var.export.tool.job.type}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_JOB_TYPE, "${tdch.export.tool.job.type}");
    runner.assertValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_FILEFORMAT, "${my.custom.var.export.tool.file.format}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_FILEFORMAT, "${tdch.export.tool.file.format}");
    runner.assertValid();
    // Assign values to the expression variables upstream in flowfile
    MockFlowFile mockFlowFile = new MockFlowFile(1L);
    Map<String, String> attributes = new HashMap<>();
    attributes.put("required_config_hive_db", "tdch");
    attributes.put("required_config_hive_table", "example7_hive");
    attributes.put("required_config_teradata_db", "finance");
    attributes.put("required_config_teradata_table", "example7_td");
    attributes.put("config_teradata_truncate_table", "true");
    attributes.put("config_hive_field_names", "h1,h2");
    attributes.put("config_teradata_field_names", "c1, c2");
    attributes.put("config_num_mappers", "5");
    attributes.put("config_throttle_mappers_flag", "true");
    attributes.put("config_hive_source_date_format", "yy-MM-dd");
    attributes.put("config_hive_source_time_format", "HH:mm");
    attributes.put("config_hive_source_timestamp_format", "yyyy-MM-dd HH:mm:ss");
    attributes.put("config_teradata_target_date_format", "yy-MM-dd");
    attributes.put("config_teradata_target_time_format", "HH:mm");
    attributes.put("config_teradata_target_timestamp_format", "yyyy-MM-dd HH:mm:ss");
    attributes.put("config_teradata_string_truncate_flag", "false");
    attributes.put("config_teradata_charset", "UTF8");
    attributes.put("config_teradata_use_xviews", "true");
    attributes.put("config_teradata_batch_size", "500");
    attributes.put("config_teradata_force_stage", "true");
    attributes.put("config_teradata_keep_stage_table", "true");
    attributes.put("tdch.export.tool.method", "batch.insert");
    attributes.put("tdch.export.tool.job.type", "hive");
    attributes.put("tdch.export.tool.file.format", "rcfile");
    mockFlowFile.putAttributes(attributes);
    runner.enqueue(mockFlowFile);
    runner.run(1);
    List<MockFlowFile> failedFlowFiles = runner.getFlowFilesForRelationship(TdchExportHiveToTeradata.REL_FAILURE);
    Assert.assertEquals(1, failedFlowFiles.size());
    runner.assertQueueEmpty();
    String expectedCommand = "hadoop jar $USERLIBTDCH com.teradata.connector.common.tool.ConnectorExportTool -libjars $LIB_JARS -Dtdch.output.teradata.truncate=true -classname \"com.teradata.jdbc.TeraDriver\" -url \"jdbc:teradata://localhost/database=finance,CHARSET=UTF8\" -username \"dbc\" -password ***** -method \"batch.insert\" -jobtype \"hive\" -fileformat \"rcfile\" -nummappers \"5\" -throttlemappers \"true\" -sourcedateformat \"yy-MM-dd\" -sourcetimeformat \"HH:mm\" -sourcetimestampformat \"yyyy-MM-dd HH:mm:ss\" -targetdateformat \"yy-MM-dd\" -targettimeformat \"HH:mm\" -targettimestampformat \"yyyy-MM-dd HH:mm:ss\" -stringtruncate \"false\" -sourcedatabase \"tdch\" -sourcetable \"example7_hive\" -sourcefieldnames \"h1,h2\" -targettable \"finance.example7_td\" -targetfieldnames \"c1,c2\" -usexviews \"true\" -batchsize \"500\" -forcestage \"true\" -keepstagetable \"true\" ";
    MockFlowFile failedFlowFile = failedFlowFiles.get(0);
    Assert.assertEquals(expectedCommand, failedFlowFile.getAttribute("tdch.export.hive.to.teradata.command"));
}