Examples with PlanFragment org.voltdb.catalog.PlanFragment used on opensource projects

Example 1 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class RealVoltDB method printDiagnosticInformation.

public static void printDiagnosticInformation(CatalogContext context, String procName, LoadedProcedureSet procSet) {
    StringBuilder sb = new StringBuilder();
    final CatalogMap<Procedure> catalogProcedures = context.database.getProcedures();
    PureJavaCrc32C crc = new PureJavaCrc32C();
    sb.append("Statements within " + procName + ": ").append("\n");
    for (final Procedure proc : catalogProcedures) {
        if (proc.getTypeName().equals(procName)) {
            for (Statement stmt : proc.getStatements()) {
                // compute hash for determinism check
                crc.reset();
                String sqlText = stmt.getSqltext();
                crc.update(sqlText.getBytes(Constants.UTF8ENCODING));
                int hash = (int) crc.getValue();
                sb.append("Statement Hash: ").append(hash);
                sb.append(", Statement SQL: ").append(sqlText);
                for (PlanFragment frag : stmt.getFragments()) {
                    byte[] planHash = Encoder.hexDecode(frag.getPlanhash());
                    long planId = ActivePlanRepository.getFragmentIdForPlanHash(planHash);
                    String stmtText = ActivePlanRepository.getStmtTextForPlanHash(planHash);
                    byte[] jsonPlan = ActivePlanRepository.planForFragmentId(planId);
                    sb.append(", Plan Fragment Id:").append(planId);
                    sb.append(", Plan Stmt Text:").append(stmtText);
                    sb.append(", Json Plan:").append(new String(jsonPlan));
                }
                sb.append("\n");
            }
        }
    }
    sb.append("Default CRUD Procedures: ").append("\n");
    for (Entry<String, Procedure> pair : context.m_defaultProcs.m_defaultProcMap.entrySet()) {
        crc.reset();
        String sqlText = DefaultProcedureManager.sqlForDefaultProc(pair.getValue());
        crc.update(sqlText.getBytes(Constants.UTF8ENCODING));
        int hash = (int) crc.getValue();
        sb.append("Statement Hash: ").append(hash);
        sb.append(", Statement SQL: ").append(sqlText);
        ProcedureRunner runner = procSet.getProcByName(pair.getValue().getTypeName());
        for (Statement stmt : runner.getCatalogProcedure().getStatements()) {
            for (PlanFragment frag : stmt.getFragments()) {
                byte[] planHash = Encoder.hexDecode(frag.getPlanhash());
                long planId = ActivePlanRepository.getFragmentIdForPlanHash(planHash);
                String stmtText = ActivePlanRepository.getStmtTextForPlanHash(planHash);
                byte[] jsonPlan = ActivePlanRepository.planForFragmentId(planId);
                sb.append(", Plan Fragment Id:").append(planId);
                sb.append(", Plan Stmt Text:").append(stmtText);
                sb.append(", Json Plan:").append(new String(jsonPlan));
            }
        }
        sb.append("\n");
    }
    hostLog.error(sb.toString());
}

Example 2 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class TestFragmentProgressUpdate method testPeakLargerThanCurr.

@SuppressWarnings("deprecation")
public void testPeakLargerThanCurr() throws Exception {
    m_ee.loadCatalog(0, m_catalog.serialize());
    int tableSize = 20000;
    int longOpthreshold = 10000;
    m_warehousedata.clearRowData();
    for (int i = 0; i < tableSize; ++i) {
        m_warehousedata.addRow(i, "name" + i, "st1", "st2", "city", "ST", "zip", 0, 0);
    }
    m_ee.loadTable(WAREHOUSE_TABLEID, m_warehousedata, 0, 0, 0, 0, false, false, WRITE_TOKEN);
    assertEquals(tableSize, m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    System.out.println("Rows loaded to table " + m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    Statement selectStmt = m_testProc.getStatements().getIgnoreCase("warehouse_join");
    PlanFragment selectBottomFrag = null;
    // delete 5000 records
    // I have no idea what's going on here, and just copy code from the above methods
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
        if (i != 0)
            selectBottomFrag = f;
        i++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    ParameterSet params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, READ_ONLY_TOKEN, false);
    // If want to see the stats, please uncomment the following line.
    // It is '8 393216 262144' on my machine.
    //System.out.println(m_ee.m_callsFromEE +" " + m_ee.m_peakMemoryInBytes + " "+ m_ee.m_currMemoryInBytes);
    assertEquals(longOpthreshold * m_ee.m_callsFromEE, m_ee.m_lastTuplesAccessed);
    assertTrue(m_ee.m_peakMemoryInBytes > m_ee.m_currMemoryInBytes);
}

Example 3 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class TestFragmentProgressUpdate method testFragmentProgressUpdate.

@SuppressWarnings("deprecation")
public void testFragmentProgressUpdate() throws Exception {
    m_ee.loadCatalog(0, m_catalog.serialize());
    int tableSize = 5001;
    int longOpthreshold = 10000;
    m_warehousedata.clearRowData();
    for (int i = 0; i < tableSize; ++i) {
        m_warehousedata.addRow(i, "name" + i, "st1", "st2", "city", "ST", "zip", 0, 0);
    }
    m_ee.loadTable(WAREHOUSE_TABLEID, m_warehousedata, 0, 0, 0, 0, false, false, WRITE_TOKEN);
    assertEquals(tableSize, m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    System.out.println("Rows loaded to table " + m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    Statement selectStmt = m_testProc.getStatements().getIgnoreCase("warehouse_select");
    PlanFragment selectBottomFrag = null;
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
        if (i != 0)
            selectBottomFrag = f;
        i++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    ParameterSet params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    // Like many fully successful operations, a single row fetch counts as 2 logical row operations,
    // one for locating the row and one for retrieving it.
    assertEquals(1, m_ee.m_callsFromEE);
    assertEquals(longOpthreshold, m_ee.m_lastTuplesAccessed);
    assertTrue(450000 < m_ee.m_currMemoryInBytes);
    assertTrue(550000 > m_ee.m_currMemoryInBytes);
    assertTrue(450000 < m_ee.m_peakMemoryInBytes);
    assertTrue(550000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
}

Example 4 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class TestFragmentProgressUpdate method testTwoUpdates.

@SuppressWarnings("deprecation")
public void testTwoUpdates() throws Exception {
    m_ee.loadCatalog(0, m_catalog.serialize());
    int tableSize = 10000;
    int longOpthreshold = 10000;
    m_warehousedata.clearRowData();
    for (int i = 0; i < tableSize; ++i) {
        m_warehousedata.addRow(i, "name" + i, "st1", "st2", "city", "ST", "zip", 0, 0);
    }
    m_ee.loadTable(WAREHOUSE_TABLEID, m_warehousedata, 0, 0, 0, 0, false, false, Long.MAX_VALUE);
    assertEquals(tableSize, m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    System.out.println("Rows loaded to table " + m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    Statement selectStmt = m_testProc.getStatements().getIgnoreCase("warehouse_select");
    PlanFragment selectBottomFrag = null;
    // delete 5000 records
    // I have no idea what's going on here, and just copy code from the above methods
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
        if (i != 0)
            selectBottomFrag = f;
        i++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    ParameterSet params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    // Like many fully successful operations, a single row fetch counts as 2 logical row operations,
    // one for locating the row and one for retrieving it.
    assertEquals(2, m_ee.m_callsFromEE);
    assertEquals(longOpthreshold * m_ee.m_callsFromEE, m_ee.m_lastTuplesAccessed);
    assertTrue(900000 < m_ee.m_currMemoryInBytes);
    assertTrue(1100000 > m_ee.m_currMemoryInBytes);
    assertTrue(900000 < m_ee.m_peakMemoryInBytes);
    assertTrue(1100000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    long previousPeakMemory = m_ee.m_peakMemoryInBytes;
    Statement deleteStmt = m_testProc.getStatements().getIgnoreCase("warehouse_del_half");
    assertNotNull(deleteStmt);
    PlanFragment deleteBottomFrag = null;
    int j = 0;
    // this kinda assumes the right order
    for (PlanFragment f : deleteStmt.getFragments()) {
        if (j != 0)
            deleteBottomFrag = f;
        j++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(deleteBottomFrag), Encoder.decodeBase64AndDecompressToBytes(deleteBottomFrag.getPlannodetree()), deleteStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(deleteBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, WRITE_TOKEN, false);
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    assertTrue(m_ee.m_callsFromEE > 2);
    // here the m_lastTuplesAccessed is just the same as threshold, since we start a new fragment
    assertEquals(longOpthreshold, m_ee.m_lastTuplesAccessed);
    assertTrue(450000 < m_ee.m_currMemoryInBytes);
    assertTrue(550000 > m_ee.m_currMemoryInBytes);
    assertTrue(450000 < m_ee.m_peakMemoryInBytes);
    assertTrue(550000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    // Although it is true, but I don't think we should compare the memory usage here.
    //assertTrue(m_ee.m_currMemoryInBytes < previousMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes < previousPeakMemory);
}

Example 5 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class ProcedureRunner method initSQLStmt.

public void initSQLStmt(SQLStmt stmt, Statement catStmt) {
    int fragCount = catStmt.getFragments().size();
    for (PlanFragment frag : catStmt.getFragments()) {
        byte[] planHash = Encoder.hexDecode(frag.getPlanhash());
        byte[] plan = Encoder.decodeBase64AndDecompressToBytes(frag.getPlannodetree());
        long id = ActivePlanRepository.loadOrAddRefPlanFragment(planHash, plan, catStmt.getSqltext());
        boolean transactional = frag.getNontransactional() == false;
        SQLStmt.Frag stmtFrag = new SQLStmt.Frag(id, planHash, transactional);
        if (fragCount == 1 || frag.getHasdependencies()) {
            stmt.aggregator = stmtFrag;
        } else {
            stmt.collector = stmtFrag;
        }
    }
    stmt.isReadOnly = catStmt.getReadonly();
    stmt.isReplicatedTableDML = catStmt.getReplicatedtabledml();
    stmt.site = m_site;
    int numStatementParamTypes = catStmt.getParameters().size();
    stmt.statementParamTypes = new byte[numStatementParamTypes];
    for (StmtParameter param : catStmt.getParameters()) {
        int index = param.getIndex();
        // Array-typed params currently only arise from in-lists.
        // Tweak the parameter's expected type accordingly.
        VoltType expectType = VoltType.get((byte) param.getJavatype());
        if (param.getIsarray()) {
            if (expectType == VoltType.STRING) {
                expectType = VoltType.INLIST_OF_STRING;
            } else {
                expectType = VoltType.INLIST_OF_BIGINT;
            }
        }
        stmt.statementParamTypes[index] = expectType.getValue();
    }
}