Examples with ParameterSet org.voltdb.ParameterSet used on opensource projects

Example 21 with ParameterSet

use of org.voltdb.ParameterSet in project voltdb by VoltDB.

the class ExecutionEngineJNI method coreExecutePlanFragments.

/**
     * @param undoToken Token identifying undo quantum for generated undo info
     * @param traceOn
     */
@Override
protected FastDeserializer coreExecutePlanFragments(final int batchIndex, final int numFragmentIds, final long[] planFragmentIds, final long[] inputDepIds, final Object[] parameterSets, DeterminismHash determinismHash, boolean[] isWriteFrags, int[] sqlCRCs, final long txnId, final long spHandle, final long lastCommittedSpHandle, long uniqueId, final long undoToken, final boolean traceOn) throws EEException {
    // plan frag zero is invalid
    assert ((numFragmentIds == 0) || (planFragmentIds[0] != 0));
    if (numFragmentIds == 0)
        return m_emptyDeserializer;
    final int batchSize = numFragmentIds;
    if (HOST_TRACE_ENABLED) {
        for (int i = 0; i < batchSize; ++i) {
            LOG.trace("Batch Executing planfragment:" + planFragmentIds[i] + ", params=" + parameterSets[i].toString());
        }
    }
    // serialize the param sets
    int allPsetSize = 0;
    for (int i = 0; i < batchSize; ++i) {
        if (parameterSets[i] instanceof ByteBuffer) {
            allPsetSize += ((ByteBuffer) parameterSets[i]).limit();
        } else {
            allPsetSize += ((ParameterSet) parameterSets[i]).getSerializedSize();
        }
    }
    clearPsetAndEnsureCapacity(allPsetSize);
    for (int i = 0; i < batchSize; ++i) {
        int paramStart = m_psetBuffer.position();
        Object param = parameterSets[i];
        if (param instanceof ByteBuffer) {
            ByteBuffer buf = (ByteBuffer) param;
            m_psetBuffer.put(buf);
        } else {
            ParameterSet pset = (ParameterSet) param;
            try {
                pset.flattenToBuffer(m_psetBuffer);
            } catch (final IOException exception) {
                throw new RuntimeException("Error serializing parameters for SQL batch element: " + i + " with plan fragment ID: " + planFragmentIds[i] + " and with params: " + pset.toJSONString(), exception);
            }
        }
        // determinismHash can be null in FragmentTask.processFragmentTask() and many tests
        if (determinismHash != null && isWriteFrags[i]) {
            determinismHash.offerStatement(sqlCRCs[i], paramStart, m_psetBuffer);
        }
    }
    // checkMaxFsSize();
    clearPerFragmentStatsAndEnsureCapacity(batchSize);
    // Execute the plan, passing a raw pointer to the byte buffers for input and output
    //Clear is destructive, do it before the native call
    FastDeserializer targetDeserializer = (batchIndex == 0) ? m_firstDeserializer : m_nextDeserializer;
    targetDeserializer.clear();
    final int errorCode = nativeExecutePlanFragments(pointer, batchIndex, numFragmentIds, planFragmentIds, inputDepIds, txnId, spHandle, lastCommittedSpHandle, uniqueId, undoToken, traceOn);
    try {
        checkErrorCode(errorCode);
        m_usingFallbackBuffer = m_fallbackBuffer != null;
        FastDeserializer fds = m_usingFallbackBuffer ? new FastDeserializer(m_fallbackBuffer) : targetDeserializer;
        assert (fds != null);
        try {
            // check if anything was changed
            m_dirty |= fds.readBoolean();
        } catch (final IOException ex) {
            LOG.error("Failed to deserialize result table" + ex);
            throw new EEException(ERRORCODE_WRONG_SERIALIZED_BYTES);
        }
        return fds;
    } finally {
        m_fallbackBuffer = null;
    }
}

Example 22 with ParameterSet

use of org.voltdb.ParameterSet in project voltdb by VoltDB.

the class ExecutionEngineIPC method sendPlanFragmentsInvocation.

private void sendPlanFragmentsInvocation(final Commands cmd, final int numFragmentIds, final long[] planFragmentIds, long[] inputDepIdsIn, final Object[] parameterSets, DeterminismHash determinismHash, boolean[] isWriteFrags, int[] sqlCRCs, final long txnId, final long spHandle, final long lastCommittedSpHandle, final long uniqueId, final long undoToken) {
    // big endian, not direct
    final FastSerializer fser = new FastSerializer();
    try {
        for (int i = 0; i < numFragmentIds; ++i) {
            Object params = parameterSets[i];
            // pset can be ByteBuffer or ParameterSet instance
            int paramStart = fser.getPosition();
            if (params instanceof ByteBuffer) {
                ByteBuffer buf = (ByteBuffer) params;
                fser.write(buf);
            } else {
                ParameterSet pset = (ParameterSet) params;
                fser.writeParameterSet(pset);
            }
            if (determinismHash != null && isWriteFrags[i]) {
                determinismHash.offerStatement(sqlCRCs[i], paramStart, fser.getContainerNoFlip().b());
            }
        }
    } catch (final IOException exception) {
        fser.discard();
        throw new RuntimeException(exception);
    }
    // if inputDepIds is null, make a bunch of dummies
    long[] inputDepIds = inputDepIdsIn;
    if (inputDepIds == null) {
        inputDepIds = new long[numFragmentIds];
        for (int i = 0; i < inputDepIds.length; i++) {
            inputDepIds[0] = -1;
        }
    }
    m_data.clear();
    do {
        m_data.putInt(cmd.m_id);
        m_data.putLong(txnId);
        m_data.putLong(spHandle);
        m_data.putLong(lastCommittedSpHandle);
        m_data.putLong(uniqueId);
        m_data.putLong(undoToken);
        m_data.put((m_perFragmentTimingEnabled ? (byte) 1 : (byte) 0));
        m_data.putInt(numFragmentIds);
        for (int i = 0; i < numFragmentIds; ++i) {
            m_data.putLong(planFragmentIds[i]);
        }
        for (int i = 0; i < numFragmentIds; ++i) {
            m_data.putLong(inputDepIds[i]);
        }
        verifyDataCapacity(m_data.position() + fser.size());
    } while (m_data.position() == 0);
    m_data.put(fser.getBuffer());
    fser.discard();
    try {
        m_data.flip();
        m_connection.write();
    } catch (final Exception e) {
        System.out.println("Exception: " + e.getMessage());
        throw new RuntimeException(e);
    }
}

Example 23 with ParameterSet

use of org.voltdb.ParameterSet in project voltdb by VoltDB.

the class TestFragmentProgressUpdate method testTwoUpdates.

@SuppressWarnings("deprecation")
public void testTwoUpdates() throws Exception {
    m_ee.loadCatalog(0, m_catalog.serialize());
    int tableSize = 10000;
    int longOpthreshold = 10000;
    m_warehousedata.clearRowData();
    for (int i = 0; i < tableSize; ++i) {
        m_warehousedata.addRow(i, "name" + i, "st1", "st2", "city", "ST", "zip", 0, 0);
    }
    m_ee.loadTable(WAREHOUSE_TABLEID, m_warehousedata, 0, 0, 0, 0, false, false, Long.MAX_VALUE);
    assertEquals(tableSize, m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    System.out.println("Rows loaded to table " + m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    Statement selectStmt = m_testProc.getStatements().getIgnoreCase("warehouse_select");
    PlanFragment selectBottomFrag = null;
    // delete 5000 records
    // I have no idea what's going on here, and just copy code from the above methods
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
        if (i != 0)
            selectBottomFrag = f;
        i++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    ParameterSet params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    // Like many fully successful operations, a single row fetch counts as 2 logical row operations,
    // one for locating the row and one for retrieving it.
    assertEquals(2, m_ee.m_callsFromEE);
    assertEquals(longOpthreshold * m_ee.m_callsFromEE, m_ee.m_lastTuplesAccessed);
    assertTrue(900000 < m_ee.m_currMemoryInBytes);
    assertTrue(1100000 > m_ee.m_currMemoryInBytes);
    assertTrue(900000 < m_ee.m_peakMemoryInBytes);
    assertTrue(1100000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    long previousPeakMemory = m_ee.m_peakMemoryInBytes;
    Statement deleteStmt = m_testProc.getStatements().getIgnoreCase("warehouse_del_half");
    assertNotNull(deleteStmt);
    PlanFragment deleteBottomFrag = null;
    int j = 0;
    // this kinda assumes the right order
    for (PlanFragment f : deleteStmt.getFragments()) {
        if (j != 0)
            deleteBottomFrag = f;
        j++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(deleteBottomFrag), Encoder.decodeBase64AndDecompressToBytes(deleteBottomFrag.getPlannodetree()), deleteStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(deleteBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, WRITE_TOKEN, false);
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    assertTrue(m_ee.m_callsFromEE > 2);
    // here the m_lastTuplesAccessed is just the same as threshold, since we start a new fragment
    assertEquals(longOpthreshold, m_ee.m_lastTuplesAccessed);
    assertTrue(450000 < m_ee.m_currMemoryInBytes);
    assertTrue(550000 > m_ee.m_currMemoryInBytes);
    assertTrue(450000 < m_ee.m_peakMemoryInBytes);
    assertTrue(550000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    // Although it is true, but I don't think we should compare the memory usage here.
    //assertTrue(m_ee.m_currMemoryInBytes < previousMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes < previousPeakMemory);
}

Example 24 with ParameterSet

use of org.voltdb.ParameterSet in project voltdb by VoltDB.

the class TestVoltMessageSerialization method testFragmentTaskWithTwoFrags.

public void testFragmentTaskWithTwoFrags() throws IOException {
    Object[] params1 = { 10, 10.1 };
    Object[] params2 = { 20, 20.2 };
    ParameterSet param_set1 = ParameterSet.fromArrayNoCopy(params1);
    ParameterSet param_set2 = ParameterSet.fromArrayNoCopy(params2);
    ByteBuffer param1_buf = ByteBuffer.allocate(param_set1.getSerializedSize());
    param_set1.flattenToBuffer(param1_buf);
    param1_buf.flip();
    ByteBuffer param2_buf = ByteBuffer.allocate(param_set2.getSerializedSize());
    param_set2.flattenToBuffer(param2_buf);
    param2_buf.flip();
    FragmentTaskMessage ft = new FragmentTaskMessage(9, 70654312, -75, 99, true, true, false);
    ft.addFragment(new byte[20], 12, param1_buf);
    ft.addFragment(new byte[20], 24, param2_buf);
    ft.setFragmentTaskType(FragmentTaskMessage.SYS_PROC_PER_PARTITION);
    FragmentTaskMessage ft2 = (FragmentTaskMessage) checkVoltMessage(ft);
    assertEquals(ft.getInitiatorHSId(), ft2.getInitiatorHSId());
    assertEquals(ft.getCoordinatorHSId(), ft2.getCoordinatorHSId());
    assertEquals(ft.getTxnId(), ft2.getTxnId());
    assertEquals(ft.getUniqueId(), ft2.getUniqueId());
    assertEquals(ft.isReadOnly(), ft2.isReadOnly());
    assertEquals(ft.isForReplay(), ft2.isForReplay());
    assertEquals(ft.getFragmentCount(), ft2.getFragmentCount());
    assertEquals(ft.isFinalTask(), ft2.isFinalTask());
    assertEquals(ft.isSysProcTask(), ft2.isSysProcTask());
    assertEquals(2, ft2.getFragmentCount());
    ParameterSet params = null;
    ByteBuffer paramData = ft2.getParameterDataForFragment(0);
    if (paramData != null) {
        params = ParameterSet.fromByteBuffer(paramData);
        assertEquals(10, params.toArray()[0]);
        assertEquals(10.1, params.toArray()[1]);
    }
    params = null;
    paramData = ft2.getParameterDataForFragment(1);
    if (paramData != null) {
        params = ParameterSet.fromByteBuffer(paramData);
        assertEquals(20, params.toArray()[0]);
        assertEquals(20.2, params.toArray()[1]);
    }
}

Example 25 with ParameterSet

use of org.voltdb.ParameterSet in project voltdb by VoltDB.

the class TestMpTransactionState method createDummyParameterSet.

ByteBuffer createDummyParameterSet() throws IOException {
    ParameterSet blah = ParameterSet.fromArrayNoCopy(new Long(4321), new Long(5678));
    ByteBuffer params = ByteBuffer.allocate(blah.getSerializedSize());
    blah.flattenToBuffer(params);
    params.flip();
    return params;
}