Examples with DriverCqlRowHandler org.apache.hop.databases.cassandra.datastax.DriverCqlRowHandler used on opensource projects

Example 1 with DriverCqlRowHandler

use of org.apache.hop.databases.cassandra.datastax.DriverCqlRowHandler in project hop by apache.

the class DriverCqlRowHandlerTest method testNextOutputRowNoQuery.

@Test
public void testNextOutputRowNoQuery() throws Exception {
    DriverKeyspace keyspace = mock(DriverKeyspace.class);
    Session session = mock(Session.class);
    DriverCqlRowHandler rowHandler = new DriverCqlRowHandler(keyspace, session, true);
    IRowMeta rowMeta = new RowMeta();
    assertNull(rowHandler.getNextOutputRow(rowMeta, null));
}

Example 2 with DriverCqlRowHandler

use of org.apache.hop.databases.cassandra.datastax.DriverCqlRowHandler in project hop by apache.

the class DriverCqlRowHandlerTest method testQueryRows.

@Test
public void testQueryRows() throws Exception {
    List<Object[]> rowList = new ArrayList<>();
    rowList.add(new Object[] { 1L, "a", 0.2d });
    rowList.add(new Object[] { 2L, "b", 42d });
    DriverKeyspace keyspace = mock(DriverKeyspace.class);
    Session session = mock(Session.class);
    ResultSet rs = mock(ResultSet.class);
    mockColumnDefinitions(rs, DataType.cint(), DataType.text(), DataType.cdouble());
    when(session.execute(anyString())).thenReturn(rs);
    Iterator<Object[]> it = rowList.iterator();
    when(rs.isExhausted()).then(invoc -> {
        return !it.hasNext();
    });
    when(rs.one()).then(invocation -> {
        Object[] rowArr = it.next();
        Row row = mock(Row.class);
        when(row.getObject(anyInt())).then(invoc -> {
            return rowArr[(int) invoc.getArguments()[0]];
        });
        when(row.getLong(0)).thenReturn((long) rowArr[0]);
        when(row.getDouble(2)).thenReturn((double) rowArr[2]);
        return row;
    });
    DriverCqlRowHandler rowHandler = new DriverCqlRowHandler(keyspace, session, true);
    IRowMeta rowMeta = new RowMeta();
    rowMeta.addValueMeta(new ValueMetaInteger("a"));
    rowMeta.addValueMeta(new ValueMetaString("b"));
    rowMeta.addValueMeta(new ValueMetaNumber("c"));
    rowHandler.newRowQuery(mock(ITransform.class), "tab", "select * from tab", null, null, mock(ILogChannel.class));
    List<Object[]> resultRows = getNextOutputRows(rowHandler, rowMeta);
    assertEquals(2, resultRows.size());
    assertEquals(2L, resultRows.get(1)[0]);
}

Example 3 with DriverCqlRowHandler

use of org.apache.hop.databases.cassandra.datastax.DriverCqlRowHandler in project hop by apache.

the class DriverCqlRowHandlerTest method testExpandCollection.

@Test
public void testExpandCollection() throws Exception {
    List<Object[]> rowList = new ArrayList<>();
    ArrayList<Long> numList = new ArrayList<>();
    numList.add(1L);
    numList.add(2L);
    numList.add(3L);
    rowList.add(new Object[] { 1L, numList });
    rowList.add(new Object[] { 2L, new ArrayList<Long>() });
    Iterator<Object[]> it = rowList.iterator();
    DriverKeyspace keyspace = mock(DriverKeyspace.class);
    Session session = mock(Session.class);
    ResultSet rs = mock(ResultSet.class);
    when(session.execute(anyString())).thenReturn(rs);
    when(rs.isExhausted()).then(invoc -> {
        return !it.hasNext();
    });
    when(rs.one()).then(invocation -> {
        Object[] rowArr = it.next();
        Row row = mock(Row.class);
        when(row.getObject(anyInt())).then(invoc -> {
            return rowArr[(int) invoc.getArguments()[0]];
        });
        when(row.getLong(0)).thenReturn((long) rowArr[0]);
        return row;
    });
    mockColumnDefinitions(rs, DataType.bigint(), DataType.list(DataType.bigint()));
    DriverCqlRowHandler rowHandler = new DriverCqlRowHandler(keyspace, session, true);
    IRowMeta rowMeta = new RowMeta();
    rowMeta.addValueMeta(new ValueMetaInteger("id"));
    rowMeta.addValueMeta(new ValueMetaNumber("nums"));
    rowHandler.newRowQuery(mock(ITransform.class), "tab", "select * from tab", null, null, mock(ILogChannel.class));
    List<Object[]> resultRows = getNextOutputRows(rowHandler, rowMeta);
    assertEquals(4, resultRows.size());
    assertEquals(1L, resultRows.get(0)[1]);
    assertEquals(2L, resultRows.get(3)[0]);
    assertNull(resultRows.get(3)[1]);
}

Example 4 with DriverCqlRowHandler

use of org.apache.hop.databases.cassandra.datastax.DriverCqlRowHandler in project hop by apache.

the class CassandraOutputTest method validateCorrectTtlFieldTest.

@Test
public void validateCorrectTtlFieldTest() {
    DriverCqlRowHandler handler = mock(DriverCqlRowHandler.class);
    doCallRealMethod().when(co).validateTtlField(any(), any());
    doCallRealMethod().when(handler).setTtlSec(anyInt());
    String ttl = "120";
    co.validateTtlField(handler, ttl);
    verify(handler, times(1)).setTtlSec(anyInt());
    verify(co, times(0)).logDebug(any());
}

Example 5 with DriverCqlRowHandler

use of org.apache.hop.databases.cassandra.datastax.DriverCqlRowHandler in project hop by apache.

the class CassandraOutput method doBatch.

protected void doBatch(List<Object[]> batch) throws Exception {
    // stopped?
    if (isStopped()) {
        logDebug(BaseMessages.getString(CassandraOutputMeta.PKG, "CassandraOutput.Message.StoppedSkippingBatch"));
        return;
    }
    // ignore empty batch
    if (batch == null || batch.isEmpty()) {
        logDebug(BaseMessages.getString(CassandraOutputMeta.PKG, "CassandraOutput.Message.SkippingEmptyBatch"));
        return;
    }
    // construct CQL/thrift batch and commit
    int size = batch.size();
    try {
        // construct CQL
        batchInsertCql = CassandraUtils.newCQLBatch(batchSize, getMeta().isUseUnloggedBatch());
        int rowsAdded = 0;
        batch = CassandraUtils.fixBatchMismatchedTypes(batch, getInputRowMeta(), cassandraMeta);
        DriverCqlRowHandler handler = (DriverCqlRowHandler) cqlHandler;
        validateTtlField(handler, options.get(CassandraUtils.BatchOptions.TTL));
        handler.setUnloggedBatch(getMeta().isUseUnloggedBatch());
        handler.batchInsert(getInputRowMeta(), batch, cassandraMeta, consistencyLevel, getMeta().isInsertFieldsNotInMeta(), getLogChannel());
        // commit
        if (data.connection == null) {
            openConnection(false);
        }
        logDetailed(BaseMessages.getString(CassandraOutputMeta.PKG, "CassandraOutput.Message.CommittingBatch", tableName, "" + rowsAdded));
    } catch (Exception e) {
        logError(e.getLocalizedMessage(), e);
        setErrors(getErrors() + 1);
        closeConnection(data.connection);
        data.connection = null;
        logDetailed(BaseMessages.getString(CassandraOutputMeta.PKG, "CassandraOutput.Error.FailedToInsertBatch", "" + size), e);
        logDetailed(BaseMessages.getString(CassandraOutputMeta.PKG, "CassandraOutput.Message.WillNowTrySplittingIntoSubBatches"));
        // is it possible to divide and conquer?
        if (size == 1) {
            // single error row - found it!
            if (getTransformMeta().isDoingErrorHandling()) {
                putError(getInputRowMeta(), batch.get(0), 1L, e.getMessage(), null, "ERR_INSERT01");
            }
        } else if (size > batchSplitFactor) {
            // split into smaller batches and try separately
            List<Object[]> subBatch = new ArrayList<>();
            while (batch.size() > batchSplitFactor) {
                while (subBatch.size() < batchSplitFactor && batch.size() > 0) {
                    // remove from the right - avoid internal shifting
                    subBatch.add(batch.remove(batch.size() - 1));
                }
                doBatch(subBatch);
                subBatch.clear();
            }
            doBatch(batch);
        } else {
            // try each row individually
            List<Object[]> subBatch = new ArrayList<>();
            while (batch.size() > 0) {
                subBatch.clear();
                // remove from the right - avoid internal shifting
                subBatch.add(batch.remove(batch.size() - 1));
                doBatch(subBatch);
            }
        }
    }
    // 
    for (Object[] row : batch) {
        putRow(getInputRowMeta(), row);
    }
}