Examples with FastDeserializer org.voltdb.messaging.FastDeserializer used on opensource projects

Example 1 with FastDeserializer

use of org.voltdb.messaging.FastDeserializer in project voltdb by VoltDB.

the class FragmentTask method processFragmentTask.

// Cut and pasted from ExecutionSite processFragmentTask(), then
// modifed to work in the new world
public FragmentResponseMessage processFragmentTask(SiteProcedureConnection siteConnection) {
    // Ensure default procs loaded here
    // Also used for LoadMultipartitionTable
    String procNameToLoad = m_fragmentMsg.getProcNameToLoad();
    if (procNameToLoad != null) {
        // this will ensure proc is loaded
        ProcedureRunner runner = siteConnection.getProcedureRunner(procNameToLoad);
        assert (runner != null);
    }
    // IZZY: actually need the "executor" HSId these days?
    final FragmentResponseMessage currentFragResponse = new FragmentResponseMessage(m_fragmentMsg, m_initiator.getHSId());
    currentFragResponse.setStatus(FragmentResponseMessage.SUCCESS, null);
    if (m_inputDeps != null) {
        siteConnection.stashWorkUnitDependencies(m_inputDeps);
    }
    if (m_fragmentMsg.isEmptyForRestart()) {
        int outputDepId = m_fragmentMsg.getOutputDepId(0);
        currentFragResponse.addDependency(new DependencyPair.BufferDependencyPair(outputDepId, m_rawDummyResponse, 0, m_rawDummyResponse.length));
        return currentFragResponse;
    }
    ProcedureRunner currRunner = siteConnection.getProcedureRunner(m_fragmentMsg.getProcedureName());
    long[] executionTimes = null;
    int succeededFragmentsCount = 0;
    if (currRunner != null) {
        currRunner.getExecutionEngine().setPerFragmentTimingEnabled(m_fragmentMsg.isPerFragmentStatsRecording());
        if (m_fragmentMsg.isPerFragmentStatsRecording()) {
            // At this point, we will execute the fragments one by one.
            executionTimes = new long[1];
        }
    }
    for (int frag = 0; frag < m_fragmentMsg.getFragmentCount(); frag++) {
        byte[] planHash = m_fragmentMsg.getPlanHash(frag);
        final int outputDepId = m_fragmentMsg.getOutputDepId(frag);
        ParameterSet params = m_fragmentMsg.getParameterSetForFragment(frag);
        final int inputDepId = m_fragmentMsg.getOnlyInputDepId(frag);
        long fragmentId = 0;
        byte[] fragmentPlan = null;
        /*
             * Currently the error path when executing plan fragments
             * does not adequately distinguish between fatal errors and
             * abort type errors that should result in a roll back.
             * Assume that it is ninja: succeeds or doesn't return.
             * No roll back support.
             *
             * AW in 2012, the preceding comment might be wrong,
             * I am pretty sure what we don't support is partial rollback.
             * The entire procedure will roll back successfully on failure
             */
        VoltTable dependency = null;
        try {
            FastDeserializer fragResult;
            fragmentPlan = m_fragmentMsg.getFragmentPlan(frag);
            String stmtText = null;
            // if custom fragment, load the plan and get local fragment id
            if (fragmentPlan != null) {
                // statement text for unplanned fragments are pulled from the message,
                // to ensure that we get the correct constants from the most recent
                // invocation.
                stmtText = m_fragmentMsg.getStmtText(frag);
                fragmentId = ActivePlanRepository.loadOrAddRefPlanFragment(planHash, fragmentPlan, null);
            } else // otherwise ask the plan source for a local fragment id
            {
                fragmentId = ActivePlanRepository.getFragmentIdForPlanHash(planHash);
                stmtText = ActivePlanRepository.getStmtTextForPlanHash(planHash);
            }
            // set up the batch context for the fragment set
            siteConnection.setBatch(m_fragmentMsg.getCurrentBatchIndex());
            fragResult = siteConnection.executePlanFragments(1, new long[] { fragmentId }, new long[] { inputDepId }, new ParameterSet[] { params }, null, // for long-running queries
            stmtText == null ? null : new String[] { stmtText }, // FragmentTasks don't generate statement hashes,
            new boolean[] { false }, null, m_txnState.txnId, m_txnState.m_spHandle, m_txnState.uniqueId, m_txnState.isReadOnly(), VoltTrace.log(VoltTrace.Category.EE) != null);
            // get a copy of the result buffers from the cache buffer so we can post the
            // fragment response to the network
            final int tableSize;
            final byte[] fullBacking;
            try {
                // read the complete size of the buffer used
                fragResult.readInt();
                // read number of dependencies (1)
                fragResult.readInt();
                // read the dependencyId() -1;
                fragResult.readInt();
                tableSize = fragResult.readInt();
                fullBacking = new byte[tableSize];
                // get a copy of the buffer
                fragResult.readFully(fullBacking);
            } catch (final IOException ex) {
                LOG.error("Failed to deserialze result table" + ex);
                throw new EEException(ExecutionEngine.ERRORCODE_WRONG_SERIALIZED_BYTES);
            }
            if (hostLog.isTraceEnabled()) {
                hostLog.l7dlog(Level.TRACE, LogKeys.org_voltdb_ExecutionSite_SendingDependency.name(), new Object[] { outputDepId }, null);
            }
            currentFragResponse.addDependency(new DependencyPair.BufferDependencyPair(outputDepId, fullBacking, 0, tableSize));
        } catch (final EEException e) {
            hostLog.l7dlog(Level.TRACE, LogKeys.host_ExecutionSite_ExceptionExecutingPF.name(), new Object[] { Encoder.hexEncode(planHash) }, e);
            currentFragResponse.setStatus(FragmentResponseMessage.UNEXPECTED_ERROR, e);
            if (currentFragResponse.getTableCount() == 0) {
                // Make sure the response has at least 1 result with a valid DependencyId
                currentFragResponse.addDependency(new DependencyPair.BufferDependencyPair(outputDepId, m_rawDummyResult, 0, m_rawDummyResult.length));
            }
            break;
        } catch (final SQLException e) {
            hostLog.l7dlog(Level.TRACE, LogKeys.host_ExecutionSite_ExceptionExecutingPF.name(), new Object[] { Encoder.hexEncode(planHash) }, e);
            currentFragResponse.setStatus(FragmentResponseMessage.UNEXPECTED_ERROR, e);
            if (currentFragResponse.getTableCount() == 0) {
                // Make sure the response has at least 1 result with a valid DependencyId
                currentFragResponse.addDependency(new DependencyPair.BufferDependencyPair(outputDepId, m_rawDummyResult, 0, m_rawDummyResult.length));
            }
            break;
        } catch (final InterruptException e) {
            hostLog.l7dlog(Level.TRACE, LogKeys.host_ExecutionSite_ExceptionExecutingPF.name(), new Object[] { Encoder.hexEncode(planHash) }, e);
            currentFragResponse.setStatus(FragmentResponseMessage.UNEXPECTED_ERROR, e);
            if (currentFragResponse.getTableCount() == 0) {
                // Make sure the response has at least 1 result with a valid DependencyId
                currentFragResponse.addDependency(new DependencyPair.BufferDependencyPair(outputDepId, m_rawDummyResult, 0, m_rawDummyResult.length));
            }
            break;
        } finally {
            // ensure adhoc plans are unloaded
            if (fragmentPlan != null) {
                ActivePlanRepository.decrefPlanFragmentById(fragmentId);
            }
            // The single-partition stored procedure handler is in the ProcedureRunner.
            if (currRunner != null) {
                succeededFragmentsCount = currRunner.getExecutionEngine().extractPerFragmentStats(1, executionTimes);
                long stmtDuration = 0;
                int stmtResultSize = 0;
                int stmtParameterSetSize = 0;
                if (m_fragmentMsg.isPerFragmentStatsRecording()) {
                    stmtDuration = executionTimes == null ? 0 : executionTimes[0];
                    stmtResultSize = dependency == null ? 0 : dependency.getSerializedSize();
                    stmtParameterSetSize = params == null ? 0 : params.getSerializedSize();
                }
                currRunner.getStatsCollector().endFragment(m_fragmentMsg.getStmtName(frag), m_fragmentMsg.isCoordinatorTask(), succeededFragmentsCount == 0, m_fragmentMsg.isPerFragmentStatsRecording(), stmtDuration, stmtResultSize, stmtParameterSetSize);
            }
        }
    }
    return currentFragResponse;
}

Example 2 with FastDeserializer

use of org.voltdb.messaging.FastDeserializer in project voltdb by VoltDB.

the class MockExecutionEngine method coreExecutePlanFragments.

@Override
protected FastDeserializer coreExecutePlanFragments(final int bufferHint, final int numFragmentIds, final long[] planFragmentIds, final long[] inputDepIds, final Object[] parameterSets, final DeterminismHash determinismHash, boolean[] isWriteFrags, int[] sqlCRCs, final long txnId, final long spHandle, final long lastCommittedSpHandle, final long uniqueId, final long undoToken, boolean traceOn) throws EEException {
    if (numFragmentIds != 1) {
        return null;
    }
    VoltTable vt;
    // TestExecutionSite uses this mock site.
    //
    // For interesting things to happen, the fuzz tester must provide a parameter
    // named 'txn_outcome'.  If this parameter is present, then the transaction
    // result will be determined by the parameter which follows 'txn_outcome'
    // according to:
    //
    // commit : every execution site will complete this transaction normally
    // rollback_all : every execution site should throw an exception for rollback
    // rollback_random : each execution site should randomly decide to rollback
    //                   This includes the final aggregating execution site call.
    ArrayList<Object> params = new ArrayList<Object>();
    // de-serialize all parameter sets
    if (parameterSets[0] instanceof ByteBuffer) {
        try {
            parameterSets[0] = ParameterSet.fromByteBuffer((ByteBuffer) parameterSets[0]);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    for (Object param : ((ParameterSet) parameterSets[0]).toArray()) {
        params.add(param);
    }
    int txn_outcome_index = params.indexOf("txn_outcome");
    if (txn_outcome_index != -1) {
        String txn_outcome = (String) params.get(txn_outcome_index + 1);
        if (txn_outcome.equals("rollback_all")) {
            //System.out.println("Throwing MASSIVE exception for rollback.");
            throwExceptionForError(ERRORCODE_ERROR);
        } else if (txn_outcome.equals("rollback_random")) {
            Random rand = new Random(System.currentTimeMillis());
            if (rand.nextInt(100) < 20) {
                //System.out.println("Throwing exception for rollback");
                //if (planFragmentId == 1)
                //{
                //    System.out.println("ROLLING BACK COORDINATOR");
                //}
                throwExceptionForError(ERRORCODE_ERROR);
            }
        }
    }
    vt = new VoltTable(new VoltTable.ColumnInfo[] { new VoltTable.ColumnInfo("foo", VoltType.INTEGER) });
    vt.addRow(Integer.valueOf(1));
    ByteBuffer buf = ByteBuffer.allocate(vt.getSerializedSize());
    vt.flattenToBuffer(buf);
    return new FastDeserializer(buf);
}

Example 3 with FastDeserializer

use of org.voltdb.messaging.FastDeserializer in project voltdb by VoltDB.

the class ExecutionEngine method executePlanFragments.

/** Run multiple plan fragments */
public FastDeserializer executePlanFragments(int numFragmentIds, long[] planFragmentIds, long[] inputDepIds, Object[] parameterSets, DeterminismHash determinismHash, String[] sqlTexts, boolean[] isWriteFrags, int[] sqlCRCs, long txnId, long spHandle, long lastCommittedSpHandle, long uniqueId, long undoQuantumToken, boolean traceOn) throws EEException {
    try {
        // For now, re-transform undoQuantumToken to readOnly. Redundancy work in site.executePlanFragments()
        m_fragmentContext = (undoQuantumToken == Long.MAX_VALUE) ? FragmentContext.RO_BATCH : FragmentContext.RW_BATCH;
        // reset context for progress updates
        m_startTime = 0;
        m_logDuration = INITIAL_LOG_DURATION;
        m_sqlTexts = sqlTexts;
        if (traceOn) {
            final VoltTrace.TraceEventBatch traceLog = VoltTrace.log(VoltTrace.Category.SPSITE);
            if (traceLog != null) {
                traceLog.add(() -> VoltTrace.beginDuration("execplanfragment", "txnId", TxnEgo.txnIdToString(txnId), "partition", Integer.toString(m_partitionId)));
            }
        }
        FastDeserializer results = coreExecutePlanFragments(m_currentBatchIndex, numFragmentIds, planFragmentIds, inputDepIds, parameterSets, determinismHash, isWriteFrags, sqlCRCs, txnId, spHandle, lastCommittedSpHandle, uniqueId, undoQuantumToken, traceOn);
        if (traceOn) {
            final VoltTrace.TraceEventBatch traceLog = VoltTrace.log(VoltTrace.Category.SPSITE);
            if (traceLog != null) {
                traceLog.add(VoltTrace::endDuration);
            }
        }
        m_plannerStats.updateEECacheStats(m_eeCacheSize, numFragmentIds - m_cacheMisses, m_cacheMisses, m_partitionId);
        return results;
    } finally {
        // don't count any cache misses when there's an exception. This is a lie and they
        // will still be used to estimate the cache size, but it's hard to count cache hits
        // during an exception, so we don't count cache misses either to get the right ratio.
        m_cacheMisses = 0;
        m_sqlTexts = null;
        m_fragmentContext = FragmentContext.UNKNOWN;
    }
}

Example 4 with FastDeserializer

use of org.voltdb.messaging.FastDeserializer in project voltdb by VoltDB.

the class ProcedureRunner method fastPath.

// Batch up pre-planned fragments, but handle ad hoc independently.
private VoltTable[] fastPath(List<QueuedSQL> batch, final boolean finalTask) {
    final int batchSize = batch.size();
    Object[] params = new Object[batchSize];
    long[] fragmentIds = new long[batchSize];
    String[] sqlTexts = new String[batchSize];
    boolean[] isWriteFrag = new boolean[batchSize];
    int[] sqlCRCs = new int[batchSize];
    int succeededFragmentsCount = 0;
    int i = 0;
    for (final QueuedSQL qs : batch) {
        assert (qs.stmt.collector == null);
        fragmentIds[i] = qs.stmt.aggregator.id;
        // use the pre-serialized params if it exists
        params[i] = qs.params;
        sqlTexts[i] = qs.stmt.getText();
        isWriteFrag[i] = !qs.stmt.isReadOnly;
        sqlCRCs[i] = SQLStmtAdHocHelper.getHash(qs.stmt);
        i++;
    }
    VoltTable[] results = null;
    // Before executing the fragments, tell the EE if this batch should be timed.
    getExecutionEngine().setPerFragmentTimingEnabled(m_perCallStats.samplingStmts());
    try {
        FastDeserializer fragResult = m_site.executePlanFragments(batchSize, fragmentIds, null, params, m_determinismHash, sqlTexts, isWriteFrag, sqlCRCs, m_txnState.txnId, m_txnState.m_spHandle, m_txnState.uniqueId, m_isReadOnly, VoltTrace.log(VoltTrace.Category.EE) != null);
        final int totalSize;
        try {
            // read the complete size of the buffer used
            totalSize = fragResult.readInt();
        } catch (final IOException ex) {
            log.error("Failed to deserialze result table" + ex);
            throw new EEException(ExecutionEngine.ERRORCODE_WRONG_SERIALIZED_BYTES);
        }
        final ByteBuffer rawDataBuff;
        if ((m_batchIndex == 0 && !m_site.usingFallbackBuffer()) || finalTask) {
            // If this is the first or final batch, skip the copy of the underlying byte array
            rawDataBuff = fragResult.buffer();
        } else {
            rawDataBuff = fragResult.readBuffer(totalSize);
        }
        results = TableHelper.convertBackedBufferToTables(rawDataBuff, batchSize);
    } catch (Throwable ex) {
        if (!m_isReadOnly) {
            // roll back the current batch and re-throw the EE exception
            m_site.truncateUndoLog(true, m_spBigBatchBeginToken >= 0 ? m_spBigBatchBeginToken : m_site.getLatestUndoToken(), m_txnState.m_spHandle, null);
        }
        throw ex;
    } finally {
        long[] executionTimes = null;
        if (m_perCallStats.samplingStmts()) {
            executionTimes = new long[batchSize];
        }
        succeededFragmentsCount = getExecutionEngine().extractPerFragmentStats(batchSize, executionTimes);
        for (i = 0; i < batchSize; i++) {
            QueuedSQL qs = batch.get(i);
            // No coordinator task for a single partition procedure.
            boolean isCoordinatorTask = false;
            // If all the fragments in this batch are executed successfully, succeededFragmentsCount == batchSize.
            // Otherwise, the fragment whose index equals succeededFragmentsCount is the one that failed.
            boolean failed = i == succeededFragmentsCount;
            m_perCallStats.recordStatementStats(qs.stmt.getStmtName(), isCoordinatorTask, failed, executionTimes == null ? 0 : executionTimes[i], results == null ? null : results[i], qs.params);
            // If this fragment failed, no subsequent fragments will be executed.
            if (failed) {
                break;
            }
        }
    }
    return results;
}

Example 5 with FastDeserializer

use of org.voltdb.messaging.FastDeserializer in project voltdb by VoltDB.

the class TestTwoSitePlans method testMultiSiteSelectAll.

public void testMultiSiteSelectAll() {
    ParameterSet params = ParameterSet.emptyParameterSet();
    int outDepId = 1 | DtxnConstants.MULTIPARTITION_DEPENDENCY;
    FastDeserializer fragResult1 = ee1.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    VoltTable dependency1 = null;
    try {
        // ignore totalsize field in message
        fragResult1.readInt();
        dependency1 = TableHelper.convertBackedBufferToTables(fragResult1.buffer(), 1)[0];
        System.out.println(dependency1.toString());
    } catch (Exception e) {
        e.printStackTrace();
    }
    assertTrue(dependency1 != null);
    FastDeserializer fragResult2 = ee2.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    VoltTable dependency2 = null;
    try {
        // ignore totalsize field in message
        fragResult2.readInt();
        dependency2 = TableHelper.convertBackedBufferToTables(fragResult2.buffer(), 1)[0];
        System.out.println(dependency2.toString());
    } catch (Exception e) {
        e.printStackTrace();
    }
    assertTrue(dependency2 != null);
    ee1.stashDependency(outDepId, dependency1);
    ee1.stashDependency(outDepId, dependency2);
    FastDeserializer fragResult3 = ee1.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectTopFrag) }, new long[] { outDepId }, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    // The underlying buffers are being reused
    assert (fragResult1.buffer() == fragResult3.buffer());
    try {
        // ignore totalsize field in message
        fragResult3.readInt();
        dependency1 = TableHelper.convertBackedBufferToTables(fragResult3.buffer(), 1)[0];
        System.out.println("Final Result");
        System.out.println(dependency1.toString());
    } catch (Exception e) {
        e.printStackTrace();
    }
}