Examples with JdbcEntryData org.apache.gobblin.converter.jdbc.JdbcEntryData used on opensource projects

Example 1 with JdbcEntryData

use of org.apache.gobblin.converter.jdbc.JdbcEntryData in project incubator-gobblin by apache.

the class JdbcBufferedInserterTestBase method createJdbcEntry.

private JdbcEntryData createJdbcEntry(Collection<String> colNames, int colSize) {
    List<JdbcEntryDatum> datumList = new ArrayList<>();
    for (String colName : colNames) {
        JdbcEntryDatum datum = new JdbcEntryDatum(colName, RandomStringUtils.randomAlphabetic(colSize));
        datumList.add(datum);
    }
    return new JdbcEntryData(datumList);
}

Example 2 with JdbcEntryData

use of org.apache.gobblin.converter.jdbc.JdbcEntryData in project incubator-gobblin by apache.

the class MySqlBufferedInserterTest method testMySqlBufferedInsert.

public void testMySqlBufferedInsert() throws SQLException {
    final int colNums = 20;
    final int batchSize = 10;
    final int entryCount = 107;
    final int colSize = 7;
    State state = new State();
    state.setProp(WRITER_JDBC_INSERT_BATCH_SIZE, Integer.toString(batchSize));
    JdbcBufferedInserter inserter = getJdbcBufferedInserter(state, conn);
    PreparedStatement pstmt = mock(PreparedStatement.class);
    when(conn.prepareStatement(anyString())).thenReturn(pstmt);
    List<JdbcEntryData> jdbcEntries = createJdbcEntries(colNums, colSize, entryCount);
    for (JdbcEntryData entry : jdbcEntries) {
        inserter.insert(db, table, entry);
    }
    inserter.flush();
    verify(conn, times(2)).prepareStatement(anyString());
    verify(pstmt, times(11)).clearParameters();
    verify(pstmt, times(11)).execute();
    verify(pstmt, times(colNums * entryCount)).setObject(anyInt(), anyObject());
    reset(pstmt);
}

Example 3 with JdbcEntryData

use of org.apache.gobblin.converter.jdbc.JdbcEntryData in project incubator-gobblin by apache.

the class MySqlBufferedInserterTest method testMySqlBufferedInsertParamLimit.

public void testMySqlBufferedInsertParamLimit() throws SQLException {
    final int colNums = 50;
    final int batchSize = 10;
    final int entryCount = 107;
    final int colSize = 3;
    final int maxParamSize = 500;
    State state = new State();
    state.setProp(WRITER_JDBC_INSERT_BATCH_SIZE, Integer.toString(batchSize));
    state.setProp(WRITER_JDBC_MAX_PARAM_SIZE, maxParamSize);
    MySqlBufferedInserter inserter = new MySqlBufferedInserter(state, conn);
    PreparedStatement pstmt = mock(PreparedStatement.class);
    when(conn.prepareStatement(anyString())).thenReturn(pstmt);
    List<JdbcEntryData> jdbcEntries = createJdbcEntries(colNums, colSize, entryCount);
    for (JdbcEntryData entry : jdbcEntries) {
        inserter.insert(db, table, entry);
    }
    inserter.flush();
    int expectedBatchSize = maxParamSize / colNums;
    int expectedExecuteCount = entryCount / expectedBatchSize + 1;
    verify(conn, times(2)).prepareStatement(anyString());
    verify(pstmt, times(expectedExecuteCount)).clearParameters();
    verify(pstmt, times(expectedExecuteCount)).execute();
    verify(pstmt, times(colNums * entryCount)).setObject(anyInt(), anyObject());
    reset(pstmt);
}

Example 4 with JdbcEntryData

use of org.apache.gobblin.converter.jdbc.JdbcEntryData in project incubator-gobblin by apache.

the class TeradataBufferedInserterTest method testTeradataBufferedInsert.

public void testTeradataBufferedInsert() throws SQLException {
    final int colNums = 20;
    final int batchSize = 10;
    final int entryCount = 107;
    final int colSize = 7;
    State state = new State();
    state.setProp(WRITER_JDBC_INSERT_BATCH_SIZE, Integer.toString(batchSize));
    JdbcBufferedInserter inserter = getJdbcBufferedInserter(state, conn);
    MockParameterMetaData mockMetadata = new MockParameterMetaData();
    mockMetadata.setParameterCount(2);
    mockMetadata.setParameterType(0, 12);
    mockMetadata.setParameterType(1, -5);
    PreparedStatement pstmt = Mockito.mock(PreparedStatement.class);
    when(pstmt.getParameterMetaData()).thenReturn(mockMetadata);
    when(pstmt.executeBatch()).thenReturn(new int[] { 1, 1, 1 });
    when(conn.prepareStatement(anyString())).thenReturn(pstmt);
    List<JdbcEntryData> jdbcEntries = createJdbcEntries(colNums, colSize, entryCount);
    for (JdbcEntryData entry : jdbcEntries) {
        inserter.insert(db, table, entry);
    }
    inserter.flush();
    verify(conn, times(2)).prepareStatement(anyString());
    verify(pstmt, times(107)).addBatch();
    verify(pstmt, times((int) Math.ceil((double) entryCount / batchSize))).executeBatch();
    verify(pstmt, times(entryCount)).clearParameters();
    verify(pstmt, times(colNums * entryCount)).setObject(anyInt(), anyObject());
    reset(pstmt);
}

Example 5 with JdbcEntryData

use of org.apache.gobblin.converter.jdbc.JdbcEntryData in project incubator-gobblin by apache.

the class MySqlBufferedInserter method insertBatch.

@Override
protected boolean insertBatch(PreparedStatement pstmt) throws SQLException {
    int i = 0;
    pstmt.clearParameters();
    for (JdbcEntryData pendingEntry : MySqlBufferedInserter.this.pendingInserts) {
        for (JdbcEntryDatum datum : pendingEntry) {
            pstmt.setObject(++i, datum.getVal());
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Executing SQL " + pstmt);
    }
    return pstmt.execute();
}