Examples with TdchConnectionService com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService used on opensource projects

Example 16 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testTeradataStagingTable.

@Test
public void testTeradataStagingTable() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals(null, runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_STAGING_TABLE_NAME).getDefaultValue());
    Assert.assertFalse(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_STAGING_TABLE_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_STAGING_TABLE_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_STAGING_TABLE, "td_staging_table");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_STAGING_TABLE, "");
    Assert.assertFalse(result.isValid());
    runner.removeProperty(TdchExportHiveToTeradata.TERADATA_STAGING_TABLE);
    runner.assertValid();
}

Example 17 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testExport_HiveTextToTeradataBatchInsert_5_6_SetRequiredAndDefaultPropertiesViaExpressionLanguageToEmptyValues.

@Test
public void testExport_HiveTextToTeradataBatchInsert_5_6_SetRequiredAndDefaultPropertiesViaExpressionLanguageToEmptyValues() throws InitializationException {
    /*
        From actual run:
        Key: 'tdch.export.hive.to.teradata.command'
	Value: 'hadoop jar $USERLIBTDCH com.teradata.connector.common.tool.ConnectorExportTool -libjars $LIB_JARS -Dtdch.output.teradata.truncate=false -classname "com.teradata.jdbc.TeraDriver" -url "jdbc:teradata://localhost/database=finance" -username "dbc" -password ***** -method "batch.insert" -jobtype "hive" -fileformat "textfile" -nummappers "2" -throttlemappers "false" -stringtruncate "true" -sourcedatabase "tdch" -sourcetable "example5_hive" -targettable "finance.example5_td" -usexviews "false" -batchsize "10000" -forcestage "false" -keepstagetable "false" '
	*/
    // This test is similar to testExport_HiveRcFileToTeradataBatchInsert_5_8_SetRequiredAndDefaultPropertiesViaExpressionLanguageToEmptyValues()
    // It differs in terms of: hive format is textfile, hive format is set to empty via expression language.
    // This test covers assigning empty values to all properties that are either required or get a default value. It ensures that all of them can be set via expression variables.
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DevTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    // These are required, and support arbitrary expression variable
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "${required_config_hive_db}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "${required_config_hive_table}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "${required_config_teradata_db}.${required_config_teradata_table}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE, "${config_teradata_truncate_table}");
    // These are optional (but get defaults when processor is instantiated), and support arbitrary expression variable
    runner.setProperty(TdchExportHiveToTeradata.HIVE_FIELD_NAMES, "${config_hive_field_names}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_FIELD_NAMES, "${config_teradata_field_names}");
    runner.setProperty(TdchExportHiveToTeradata.NUMBER_OF_MAPPERS, "${config_num_mappers}");
    runner.setProperty(TdchExportHiveToTeradata.THROTTLE_MAPPERS_FLAG, "${config_throttle_mappers_flag}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_SOURCE_DATE_FORMAT, "${config_hive_source_date_format}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_SOURCE_TIME_FORMAT, "${config_hive_source_time_format}");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_SOURCE_TIMESTAMP_FORMAT, "${config_hive_source_timestamp_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TARGET_DATE_FORMAT, "${config_teradata_target_date_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TARGET_TIME_FORMAT, "${config_teradata_target_time_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_TARGET_TIMESTAMP_FORMAT, "${config_teradata_target_timestamp_format}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_STRING_TRUNCATE_FLAG, "${config_teradata_string_truncate_flag}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_CHARSET, "${config_teradata_charset}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "${config_teradata_use_xviews}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_BATCH_SIZE, "${config_teradata_batch_size}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_FORCE_STAGE, "${config_teradata_force_stage}");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_KEEP_STAGE_TABLE, "${config_teradata_keep_stage_table}");
    runner.assertValid();
    // These need the specific expression language variable since processor checks them. The test verifies this.
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_EXPORT_TOOL_METHOD, "${my.custom.var.export.tool.method}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_EXPORT_TOOL_METHOD, "${tdch.export.tool.method}");
    runner.assertValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_JOB_TYPE, "${my.custom.var.export.tool.job.type}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_JOB_TYPE, "${tdch.export.tool.job.type}");
    runner.assertValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_FILEFORMAT, "${my.custom.var.export.tool.file.format}");
    runner.assertNotValid();
    runner.setProperty(TdchExportHiveToTeradata.HIVE_EXPORT_TOOL_FILEFORMAT, "${tdch.export.tool.file.format}");
    runner.assertValid();
    // Assign values to the expression variables upstream in flowfile
    MockFlowFile mockFlowFile = new MockFlowFile(1L);
    Map<String, String> attributes = new HashMap<>();
    attributes.put("required_config_hive_db", "tdch");
    attributes.put("required_config_hive_table", "example5_hive");
    attributes.put("required_config_teradata_db", "finance");
    attributes.put("required_config_teradata_table", "example5_td");
    attributes.put("config_teradata_truncate_table", "");
    attributes.put("config_hive_field_names", "");
    attributes.put("config_teradata_field_names", "");
    attributes.put("config_num_mappers", "");
    attributes.put("config_throttle_mappers_flag", "");
    attributes.put("config_hive_source_date_format", "");
    attributes.put("config_hive_source_time_format", "");
    attributes.put("config_hive_source_timestamp_format", "");
    attributes.put("config_teradata_target_date_format", "");
    attributes.put("config_teradata_target_time_format", "");
    attributes.put("config_teradata_target_timestamp_format", "");
    attributes.put("config_teradata_string_truncate_flag", "");
    attributes.put("config_teradata_charset", "");
    attributes.put("config_teradata_use_xviews", "");
    attributes.put("config_teradata_batch_size", "");
    attributes.put("config_teradata_force_stage", "");
    attributes.put("config_teradata_keep_stage_table", "");
    attributes.put("tdch.export.tool.method", "");
    attributes.put("tdch.export.tool.job.type", "");
    attributes.put("tdch.export.tool.file.format", "");
    mockFlowFile.putAttributes(attributes);
    runner.enqueue(mockFlowFile);
    runner.run(1);
    List<MockFlowFile> failedFlowFiles = runner.getFlowFilesForRelationship(TdchExportHiveToTeradata.REL_FAILURE);
    Assert.assertEquals(1, failedFlowFiles.size());
    runner.assertQueueEmpty();
    String expectedCommand = "hadoop jar $USERLIBTDCH com.teradata.connector.common.tool.ConnectorExportTool -libjars $LIB_JARS -Dtdch.output.teradata.truncate=false -classname \"com.teradata.jdbc.TeraDriver\" -url \"jdbc:teradata://localhost/database=finance\" -username \"dbc\" -password ***** -method \"batch.insert\" -jobtype \"hive\" -fileformat \"textfile\" -nummappers \"2\" -throttlemappers \"false\" -stringtruncate \"true\" -sourcedatabase \"tdch\" -sourcetable \"example5_hive\" -targettable \"finance.example5_td\" -usexviews \"false\" -batchsize \"10000\" -forcestage \"false\" -keepstagetable \"false\" ";
    MockFlowFile failedFlowFile = failedFlowFiles.get(0);
    Assert.assertEquals(expectedCommand, failedFlowFile.getAttribute("tdch.export.hive.to.teradata.command"));
}

Example 18 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testTeradataDatabaseTable.

@Test
public void testTeradataDatabaseTable() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals(null, runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE_NAME).getDefaultValue());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "database");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "table");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "database.");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, ".table");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "database.table");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    Assert.assertTrue(result.isValid());
    runner.assertValid();
    runner.removeProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE);
    runner.assertNotValid();
}

Example 19 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testTeradataUseXviews.

@Test
public void testTeradataUseXviews() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals("false", runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS_NAME).getDefaultValue());
    Assert.assertFalse(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "true");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "not-boolean-value");
    Assert.assertFalse(result.isValid());
    runner.removeProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS);
    runner.assertValid();
}

Example 20 with TdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.api.TdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testHiveLineSeparator.

@Test
public void testHiveLineSeparator() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals(null, runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR_NAME).getDefaultValue());
    Assert.assertFalse(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR, "/\\r");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR, "");
    Assert.assertFalse(result.isValid());
    runner.removeProperty(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR);
    runner.assertValid();
}