Examples with DummyTdchConnectionService com.thinkbiganalytics.kylo.nifi.teradata.tdch.core.controllerservice.DummyTdchConnectionService used on opensource projects

Example 16 with DummyTdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.core.controllerservice.DummyTdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testFullExport.

@Test
public void testFullExport() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    runner.enqueue();
    runner.run(1);
    List<MockFlowFile> failedFlowFiles = runner.getFlowFilesForRelationship(TdchExportHiveToTeradata.REL_FAILURE);
    Assert.assertEquals(1, failedFlowFiles.size());
    runner.assertQueueEmpty();
    String expectedCommand = "hadoop jar $USERLIBTDCH com.teradata.connector.common.tool.ConnectorExportTool -libjars $LIB_JARS -Dtdch.output.teradata.truncate=false -classname \"jdbc.driver.class.name\" -url \"jdbc.connection.url/database=teradata_db\" -username \"user.name\" -password ***** -method \"batch.insert\" -jobtype \"hive\" -fileformat \"textfile\" -nummappers \"2\" -throttlemappers \"false\" -sourcedateformat \"yyyy-MM-dd\" -sourcetimeformat \"HH:mm:ss\" -sourcetimestampformat \"yyyy-MM-dd HH:mm:ss.SSS\" -targetdateformat \"yyyy-MM-dd\" -targettimeformat \"HH:mm:ss\" -targettimestampformat \"yyyy-MM-dd HH:mm:ss.SSS\" -stringtruncate \"true\" -sourcedatabase \"hive_db\" -sourcetable \"hive_table\" -targettable \"teradata_db.teradata_table\" -usexviews \"false\" -batchsize \"10000\" -forcestage \"false\" -keepstagetable \"false\" ";
    MockFlowFile failedFlowFile = failedFlowFiles.get(0);
    Assert.assertEquals(expectedCommand, failedFlowFile.getAttribute("tdch.export.hive.to.teradata.command"));
    Assert.assertNotEquals("0", failedFlowFile.getAttribute("tdch.export.hive.to.teradata.kylo.result.code"));
    Assert.assertEquals("-1", failedFlowFile.getAttribute("tdch.export.hive.to.teradata.input.record.count"));
    Assert.assertEquals("-1", failedFlowFile.getAttribute("tdch.export.hive.to.teradata.output.record.count"));
    Assert.assertEquals("-1", failedFlowFile.getAttribute("tdch.export.hive.to.teradata.tdch.exit.code"));
    Assert.assertEquals("Unable to determine time taken", failedFlowFile.getAttribute("tdch.export.hive.to.teradata.tdch.time.taken"));
}

Example 17 with DummyTdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.core.controllerservice.DummyTdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testTeradataTruncate.

@Test
public void testTeradataTruncate() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals("false", runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE_NAME).getDefaultValue());
    Assert.assertFalse(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE, "true");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE, "");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE, "not-boolean-value");
    Assert.assertFalse(result.isValid());
    runner.removeProperty(TdchExportHiveToTeradata.TERADATA_TRUNCATE_TABLE);
    runner.assertValid();
}

Example 18 with DummyTdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.core.controllerservice.DummyTdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testTeradataDatabaseTable.

@Test
public void testTeradataDatabaseTable() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals(null, runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE_NAME).getDefaultValue());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "database");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "table");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "database.");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, ".table");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "database.table");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    Assert.assertTrue(result.isValid());
    runner.assertValid();
    runner.removeProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE);
    runner.assertNotValid();
}

Example 19 with DummyTdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.core.controllerservice.DummyTdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testTeradataUseXviews.

@Test
public void testTeradataUseXviews() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals("false", runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS_NAME).getDefaultValue());
    Assert.assertFalse(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "true");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "");
    Assert.assertFalse(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS, "not-boolean-value");
    Assert.assertFalse(result.isValid());
    runner.removeProperty(TdchExportHiveToTeradata.TERADATA_USE_XVIEWS);
    runner.assertValid();
}

Example 20 with DummyTdchConnectionService

use of com.thinkbiganalytics.kylo.nifi.teradata.tdch.core.controllerservice.DummyTdchConnectionService in project kylo by Teradata.

the class TdchExportHiveToTeradataTest method testHiveLineSeparator.

@Test
public void testHiveLineSeparator() throws InitializationException {
    final TestRunner runner = TestRunners.newTestRunner(TdchExportHiveToTeradata.class);
    TdchConnectionService tdchConnectionService = new DummyTdchConnectionService();
    runner.addControllerService(CONNECTION_SERVICE_ID, tdchConnectionService);
    runner.assertValid(tdchConnectionService);
    runner.enableControllerService(tdchConnectionService);
    runner.setProperty(TdchExportHiveToTeradata.TDCH_CONNECTION_SERVICE, CONNECTION_SERVICE_ID);
    runner.setProperty(TdchExportHiveToTeradata.HIVE_DATABASE, "hive_db");
    runner.setProperty(TdchExportHiveToTeradata.HIVE_TABLE, "hive_table");
    runner.setProperty(TdchExportHiveToTeradata.TERADATA_DATABASE_TABLE, "teradata_db.teradata_table");
    runner.assertValid();
    Assert.assertEquals(null, runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR_NAME).getDefaultValue());
    Assert.assertFalse(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR_NAME).isRequired());
    Assert.assertTrue(runner.getProcessor().getPropertyDescriptor(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR_NAME).isExpressionLanguageSupported());
    ValidationResult result = runner.setProperty(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR, "/\\r");
    Assert.assertTrue(result.isValid());
    result = runner.setProperty(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR, "");
    Assert.assertFalse(result.isValid());
    runner.removeProperty(TdchExportHiveToTeradata.HIVE_LINE_SEPARATOR);
    runner.assertValid();
}