Examples with DBCPService org.apache.nifi.dbcp.DBCPService used on opensource projects

Example 21 with DBCPService

use of org.apache.nifi.dbcp.DBCPService in project nifi by apache.

the class TestListDatabaseTables method testListTablesMultipleRefresh.

@Test
public void testListTablesMultipleRefresh() throws Exception {
    assumeFalse(isWindowsEnvironment());
    // load test data to database
    final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection();
    Statement stmt = con.createStatement();
    try {
        stmt.execute("drop table TEST_TABLE1");
        stmt.execute("drop table TEST_TABLE2");
    } catch (final SQLException sqle) {
    // Do nothing, may not have existed
    }
    stmt.execute("create table TEST_TABLE1 (id integer not null, val1 integer, val2 integer, constraint my_pk1 primary key (id))");
    stmt.execute("insert into TEST_TABLE1 (id, val1, val2) VALUES (0, NULL, 1)");
    stmt.execute("insert into TEST_TABLE1 (id, val1, val2) VALUES (1, 1, 1)");
    runner.setProperty(ListDatabaseTables.INCLUDE_COUNT, "true");
    runner.setProperty(ListDatabaseTables.REFRESH_INTERVAL, "200 millis");
    runner.run();
    runner.assertTransferCount(ListDatabaseTables.REL_SUCCESS, 1);
    List<MockFlowFile> results = runner.getFlowFilesForRelationship(ListDatabaseTables.REL_SUCCESS);
    assertEquals("2", results.get(0).getAttribute(ListDatabaseTables.DB_TABLE_COUNT));
    runner.clearTransferState();
    // Add another table immediately, the first table should not be listed again but the second should
    stmt.execute("create table TEST_TABLE2 (id integer not null, val1 integer, val2 integer, constraint my_pk2 primary key (id))");
    stmt.close();
    runner.run();
    runner.assertTransferCount(ListDatabaseTables.REL_SUCCESS, 1);
    results = runner.getFlowFilesForRelationship(ListDatabaseTables.REL_SUCCESS);
    assertEquals("0", results.get(0).getAttribute(ListDatabaseTables.DB_TABLE_COUNT));
    runner.clearTransferState();
    // Now wait longer than the refresh interval and assert the refresh has happened (i.e. the two tables are re-listed)
    Thread.sleep(500);
    runner.run();
    runner.assertTransferCount(ListDatabaseTables.REL_SUCCESS, 2);
}

Example 22 with DBCPService

use of org.apache.nifi.dbcp.DBCPService in project nifi by apache.

the class TestListDatabaseTables method setUp.

@Before
public void setUp() throws Exception {
    processor = new ListDatabaseTables();
    final DBCPService dbcp = new DBCPServiceSimpleImpl();
    final Map<String, String> dbcpProperties = new HashMap<>();
    runner = TestRunners.newTestRunner(ListDatabaseTables.class);
    runner.addControllerService("dbcp", dbcp, dbcpProperties);
    runner.enableControllerService(dbcp);
    runner.setProperty(ListDatabaseTables.DBCP_SERVICE, "dbcp");
}

Example 23 with DBCPService

use of org.apache.nifi.dbcp.DBCPService in project nifi by apache.

the class TestListDatabaseTables method testListTablesAfterRefresh.

@Test
public void testListTablesAfterRefresh() throws Exception {
    assumeFalse(isWindowsEnvironment());
    // load test data to database
    final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection();
    Statement stmt = con.createStatement();
    try {
        stmt.execute("drop table TEST_TABLE1");
        stmt.execute("drop table TEST_TABLE2");
    } catch (final SQLException sqle) {
    // Do nothing, may not have existed
    }
    stmt.execute("create table TEST_TABLE1 (id integer not null, val1 integer, val2 integer, constraint my_pk1 primary key (id))");
    stmt.execute("insert into TEST_TABLE1 (id, val1, val2) VALUES (0, NULL, 1)");
    stmt.execute("insert into TEST_TABLE1 (id, val1, val2) VALUES (1, 1, 1)");
    stmt.execute("create table TEST_TABLE2 (id integer not null, val1 integer, val2 integer, constraint my_pk2 primary key (id))");
    stmt.close();
    runner.setProperty(ListDatabaseTables.INCLUDE_COUNT, "true");
    runner.setProperty(ListDatabaseTables.REFRESH_INTERVAL, "100 millis");
    runner.run();
    runner.assertTransferCount(ListDatabaseTables.REL_SUCCESS, 2);
    List<MockFlowFile> results = runner.getFlowFilesForRelationship(ListDatabaseTables.REL_SUCCESS);
    assertEquals("2", results.get(0).getAttribute(ListDatabaseTables.DB_TABLE_COUNT));
    assertEquals("0", results.get(1).getAttribute(ListDatabaseTables.DB_TABLE_COUNT));
    runner.clearTransferState();
    runner.run();
    runner.assertTransferCount(ListDatabaseTables.REL_SUCCESS, 0);
    // Now wait longer than 100 millis and assert the refresh has happened (the two tables are re-listed)
    Thread.sleep(200);
    runner.run();
    runner.assertTransferCount(ListDatabaseTables.REL_SUCCESS, 2);
}

Example 24 with DBCPService

use of org.apache.nifi.dbcp.DBCPService in project nifi by apache.

the class TestPutHiveQL method testFailAtBeginningRollbackOnFailure.

@Test
public void testFailAtBeginningRollbackOnFailure() throws InitializationException, ProcessException, SQLException, IOException {
    final TestRunner runner = TestRunners.newTestRunner(PutHiveQL.class);
    final File tempDir = folder.getRoot();
    final File dbDir = new File(tempDir, "db");
    final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath());
    runner.addControllerService("dbcp", service);
    runner.enableControllerService(service);
    runner.setProperty(RollbackOnFailure.ROLLBACK_ON_FAILURE, "true");
    try (final Connection conn = service.getConnection()) {
        try (final Statement stmt = conn.createStatement()) {
            stmt.executeUpdate(createPersonsAutoId);
        }
    }
    runner.setProperty(PutHiveQL.HIVE_DBCP_SERVICE, "dbcp");
    // intentionally wrong syntax
    runner.enqueue("INSERT INTO PERSONS".getBytes());
    runner.enqueue("INSERT INTO PERSONS (NAME, CODE) VALUES ('Tom', 3)".getBytes());
    runner.enqueue("INSERT INTO PERSONS (NAME, CODE) VALUES ('Harry', 44)".getBytes());
    try {
        runner.run();
        fail("ProcessException should be thrown");
    } catch (AssertionError e) {
        assertTrue(e.getCause() instanceof ProcessException);
    }
    assertEquals(3, runner.getQueueSize().getObjectCount());
    runner.assertTransferCount(PutHiveQL.REL_FAILURE, 0);
    runner.assertTransferCount(PutHiveQL.REL_SUCCESS, 0);
}

Example 25 with DBCPService

use of org.apache.nifi.dbcp.DBCPService in project nifi by apache.

the class TestPutHiveQL method testRetryableFailureRollbackOnFailure.

@Test
public void testRetryableFailureRollbackOnFailure() throws InitializationException, ProcessException, SQLException, IOException {
    final TestRunner runner = TestRunners.newTestRunner(PutHiveQL.class);
    final DBCPService service = new SQLExceptionService(null);
    runner.addControllerService("dbcp", service);
    runner.enableControllerService(service);
    runner.setProperty(PutHiveQL.HIVE_DBCP_SERVICE, "dbcp");
    runner.setProperty(RollbackOnFailure.ROLLBACK_ON_FAILURE, "true");
    final String sql = "INSERT INTO PERSONS (ID, NAME, CODE) VALUES (?, ?, ?); " + "UPDATE PERSONS SET NAME='George' WHERE ID=?; ";
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("hiveql.args.1.type", String.valueOf(Types.INTEGER));
    attributes.put("hiveql.args.1.value", "1");
    attributes.put("hiveql.args.2.type", String.valueOf(Types.VARCHAR));
    attributes.put("hiveql.args.2.value", "Mark");
    attributes.put("hiveql.args.3.type", String.valueOf(Types.INTEGER));
    attributes.put("hiveql.args.3.value", "84");
    attributes.put("hiveql.args.4.type", String.valueOf(Types.INTEGER));
    attributes.put("hiveql.args.4.value", "1");
    runner.enqueue(sql.getBytes(), attributes);
    try {
        runner.run();
        fail("Should throw ProcessException");
    } catch (AssertionError e) {
        assertTrue(e.getCause() instanceof ProcessException);
    }
    assertEquals(1, runner.getQueueSize().getObjectCount());
    runner.assertAllFlowFilesTransferred(PutHiveQL.REL_RETRY, 0);
}