Examples with VoltBulkLoader org.voltdb.client.VoltBulkLoader.VoltBulkLoader used on opensource projects

Example 1 with VoltBulkLoader

use of org.voltdb.client.VoltBulkLoader.VoltBulkLoader in project voltdb by VoltDB.

the class TestVoltBulkLoader method test_Interface.

public void test_Interface(String my_schema, Object[][] my_data, int my_batchSize, ArrayList<Integer> expectedFailList, int flushInterval, boolean upsert) throws Exception {
    try {
        pathToCatalog = Configuration.getPathToCatalogForTest("vbl.jar");
        pathToDeployment = Configuration.getPathToCatalogForTest("vbl.xml");
        builder = new VoltProjectBuilder();
        builder.addLiteralSchema(my_schema);
        builder.addPartitionInfo("BLAH", "clm_integer");
        boolean success = builder.compile(pathToCatalog, 2, 1, 0);
        assertTrue(success);
        MiscUtils.copyFile(builder.getPathToDeployment(), pathToDeployment);
        config = new VoltDB.Configuration();
        config.m_pathToCatalog = pathToCatalog;
        config.m_pathToDeployment = pathToDeployment;
        localServer = new ServerThread(config);
        client1 = null;
        localServer.start();
        localServer.waitForInitialization();
        client1 = ClientFactory.createClient();
        client1.createConnection("localhost");
        prepare();
        TestFailureCallback testCallback = new TestFailureCallback();
        VoltBulkLoader bulkLoader = client1.getNewBulkLoader("BLAH", my_batchSize, upsert, testCallback);
        if (flushInterval > 0) {
            bulkLoader.setFlushInterval(0, flushInterval);
        }
        // do the test
        VoltTable modCount;
        modCount = client1.callProcedure("@AdHoc", "SELECT * FROM BLAH;").getResults()[0];
        System.out.println("data inserted to table BLAH:\n" + modCount);
        // Call validate partitioning to check if we are good.
        VoltTable valTable;
        valTable = client1.callProcedure("@ValidatePartitioning", null, null).getResults()[0];
        System.out.println("Validate for BLAH:\n" + valTable);
        while (valTable.advanceRow()) {
            long miscnt = valTable.getLong("MISPARTITIONED_ROWS");
            assertEquals(miscnt, 0);
        }
        int rowCnt = 1;
        try {
            for (Object[] nextRow : my_data) {
                Integer rowId = new Integer(rowCnt);
                bulkLoader.insertRow(rowId, nextRow);
                rowCnt++;
                if (flushInterval <= 0 && (rnd.nextInt() % 30 == 0)) {
                    //  Randomly inject a flush if no timer flush is involved.
                    bulkLoader.flush();
                }
            }
        } catch (Exception e) {
            System.err.print(e.getMessage());
        }
        System.out.println(String.format("Attempted inserting %d rows", --rowCnt));
        if (flushInterval <= 0 && rnd.nextBoolean()) {
            // One in 10 tests generate a sync and VoltBulkLoader internal state verification
            bulkLoader.drain();
            assertEquals(0, bulkLoader.getOutstandingRowCount());
            assertEquals(rowCnt, bulkLoader.getCompletedRowCount());
        }
        if (flushInterval > 0) {
            //Lets get timerFlush in
            Thread.sleep(flushInterval + 500);
            bulkLoader.drain();
            //We should have everything processed callbacked.
            assertEquals(0, bulkLoader.getOutstandingRowCount());
            assertEquals(rowCnt, bulkLoader.getCompletedRowCount());
        }
        bulkLoader.close();
        assertEquals(rowCnt, bulkLoader.getCompletedRowCount());
        assertTrue(testCallback.failureRowListMatches(expectedFailList));
    } finally {
        if (client1 != null)
            client1.close();
        client1 = null;
        if (localServer != null) {
            localServer.shutdown();
            localServer.join();
        }
        localServer = null;
        // no clue how helpful this is
        System.gc();
    }
}

Example 2 with VoltBulkLoader

use of org.voltdb.client.VoltBulkLoader.VoltBulkLoader in project voltdb by VoltDB.

the class TestVoltBulkLoader method test_multiplexing.

public void test_multiplexing(String my_schema, boolean multipleClients, boolean multipleLoaders, boolean multiPartTable, String my_tableName1, Object[][] my_data1, int my_batchSize1, ArrayList<Integer> expectedFailList1, boolean abort1, boolean upsert1, String my_tableName2, Object[][] my_data2, int my_batchSize2, ArrayList<Integer> expectedFailList2, boolean abort2, boolean upsert2) throws Exception {
    try {
        pathToCatalog = Configuration.getPathToCatalogForTest("vbl.jar");
        pathToDeployment = Configuration.getPathToCatalogForTest("vbl.xml");
        builder = new VoltProjectBuilder();
        builder.addLiteralSchema(my_schema);
        boolean sameTable = my_tableName1.equals(my_tableName2);
        assert (!(abort1 && abort2));
        if (abort1 || abort2)
            // No point in testing abort with a single loader
            assert (multipleLoaders);
        if (!multiPartTable) {
            builder.addPartitionInfo(my_tableName1, "clm_integer");
            if (!sameTable)
                builder.addPartitionInfo(my_tableName2, "clm_integer");
        }
        boolean success = builder.compile(pathToCatalog, 2, 1, 0);
        assertTrue(success);
        MiscUtils.copyFile(builder.getPathToDeployment(), pathToDeployment);
        config = new VoltDB.Configuration();
        config.m_pathToCatalog = pathToCatalog;
        config.m_pathToDeployment = pathToDeployment;
        localServer = new ServerThread(config);
        client1 = null;
        client2 = null;
        localServer.start();
        localServer.waitForInitialization();
        client1 = ClientFactory.createClient();
        client1.createConnection("localhost");
        if (multipleClients) {
            client2 = ClientFactory.createClient();
            client2.createConnection("localhost");
            // This is implicit
            multipleLoaders = true;
        } else
            client2 = client1;
        prepare();
        TestFailureCallback testCallback1 = new TestFailureCallback();
        TestFailureCallback testCallback2 = new TestFailureCallback();
        VoltBulkLoader bulkLoader1 = client1.getNewBulkLoader(my_tableName1, my_batchSize1, upsert1, testCallback1);
        VoltBulkLoader bulkLoader2;
        if (multipleLoaders) {
            bulkLoader2 = client2.getNewBulkLoader(my_tableName2, my_batchSize2, upsert2, testCallback2);
            if (!multipleClients && sameTable) {
                assert (bulkLoader1.getMaxBatchSize() == Math.min(my_batchSize1, my_batchSize2));
                assert (bulkLoader1.getMaxBatchSize() == bulkLoader2.getMaxBatchSize());
            }
        } else
            bulkLoader2 = bulkLoader1;
        // do the test
        VoltTable modCount1;
        modCount1 = client1.callProcedure("@AdHoc", "SELECT * FROM " + my_tableName1 + ";").getResults()[0];
        System.out.println("data inserted to table " + my_tableName1 + ":\n" + modCount1);
        // Call validate partitioning to check if we are good.
        VoltTable valTable1;
        valTable1 = client1.callProcedure("@ValidatePartitioning", null, null).getResults()[0];
        System.out.println("Validate for " + my_tableName1 + ":\n" + valTable1);
        while (valTable1.advanceRow()) {
            long miscnt = valTable1.getLong("MISPARTITIONED_ROWS");
            assertEquals(miscnt, 0);
        }
        if (multipleClients) {
            VoltTable modCount2;
            modCount2 = client2.callProcedure("@AdHoc", "SELECT * FROM " + my_tableName2 + ";").getResults()[0];
            System.out.println("data inserted to table " + my_tableName2 + ":\n" + modCount2);
            // Call validate partitioning to check if we are good.
            VoltTable valTable2;
            valTable2 = client2.callProcedure("@ValidatePartitioning", null, null).getResults()[0];
            System.out.println("Validate for " + my_tableName1 + ":\n" + valTable2);
            while (valTable2.advanceRow()) {
                long miscnt = valTable2.getLong("MISPARTITIONED_ROWS");
                assertEquals(miscnt, 0);
            }
        }
        int rowCnt1 = 1;
        int rowCnt2 = 1;
        try {
            while (rowCnt1 <= my_data1.length || rowCnt2 <= my_data2.length) {
                if (rowCnt1 <= my_data1.length) {
                    Integer rowId = new Integer(rowCnt1);
                    bulkLoader1.insertRow(rowId, my_data1[rowCnt1 - 1]);
                    rowCnt1++;
                //                      if (rnd.nextInt() % 30 == 0)
                //                          //  Randomly inject a flush
                //                          bulkLoader1.flush();
                }
                if (rowCnt2 <= my_data2.length) {
                    Integer rowId = new Integer(rowCnt2);
                    bulkLoader2.insertRow(rowId, my_data2[rowCnt2 - 1]);
                    rowCnt2++;
                //                      if (rnd.nextInt() % 30 == 0)
                //                          //  Randomly inject a flush
                //                          bulkLoader2.flush();
                }
            }
        } catch (Exception e) {
            System.err.print(e.getMessage());
        }
        System.out.println(String.format("Attempted inserting %d rows in Table %s and %d rows in Table %s", --rowCnt1, my_tableName1, --rowCnt2, my_tableName2));
        if (!abort1 && rnd.nextInt() % 4 == 0) {
            // One in 4 tests generate a sync and VoltBulkLoader internal state verification
            bulkLoader1.drain();
            assert (bulkLoader1.getOutstandingRowCount() == 0);
            assert (bulkLoader1.getCompletedRowCount() == rowCnt1);
        }
        if (multipleLoaders && !abort2 && rnd.nextInt() % 4 == 0) {
            bulkLoader2.drain();
            assert (bulkLoader2.getOutstandingRowCount() == 0);
            assert (bulkLoader2.getCompletedRowCount() == rowCnt2);
        }
        bulkLoader1.close();
        if (multipleLoaders)
            bulkLoader2.close();
        assert (abort1 || testCallback1.failureRowListMatches(expectedFailList1));
        assert (abort2 || testCallback2.failureRowListMatches(expectedFailList2));
    } finally {
        if (client1 != null) {
            client1.close();
            if (multipleClients && client2 != null) {
                client2.close();
                client2 = null;
            }
            client1 = null;
        }
        if (localServer != null) {
            localServer.shutdown();
            localServer.join();
        }
        localServer = null;
        // no clue how helpful this is
        System.gc();
    }
}