Examples with PlanFragment org.voltdb.catalog.PlanFragment used on opensource projects

Example 6 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class ProcedureRunner method initSQLStmt.

public void initSQLStmt(SQLStmt stmt, Statement catStmt) {
    int fragCount = catStmt.getFragments().size();
    for (PlanFragment frag : catStmt.getFragments()) {
        byte[] planHash = Encoder.hexDecode(frag.getPlanhash());
        byte[] plan = Encoder.decodeBase64AndDecompressToBytes(frag.getPlannodetree());
        long id = ActivePlanRepository.loadOrAddRefPlanFragment(planHash, plan, catStmt.getSqltext());
        boolean transactional = frag.getNontransactional() == false;
        SQLStmt.Frag stmtFrag = new SQLStmt.Frag(id, planHash, transactional);
        if (fragCount == 1 || frag.getHasdependencies()) {
            stmt.aggregator = stmtFrag;
        } else {
            stmt.collector = stmtFrag;
        }
    }
    stmt.isReadOnly = catStmt.getReadonly();
    stmt.isReplicatedTableDML = catStmt.getReplicatedtabledml();
    stmt.site = m_site;
    int numStatementParamTypes = catStmt.getParameters().size();
    stmt.statementParamTypes = new byte[numStatementParamTypes];
    for (StmtParameter param : catStmt.getParameters()) {
        int index = param.getIndex();
        // Array-typed params currently only arise from in-lists.
        // Tweak the parameter's expected type accordingly.
        VoltType expectType = VoltType.get((byte) param.getJavatype());
        if (param.getIsarray()) {
            if (expectType == VoltType.STRING) {
                expectType = VoltType.INLIST_OF_STRING;
            } else {
                expectType = VoltType.INLIST_OF_BIGINT;
            }
        }
        stmt.statementParamTypes[index] = expectType.getValue();
    }
}

Example 7 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class TestTwoSitePlans method setUp.

@SuppressWarnings("deprecation")
@Override
public void setUp() throws IOException, InterruptedException {
    VoltDB.instance().readBuildInfo("Test");
    // compile a catalog
    String testDir = BuildDirectoryUtils.getBuildDirectoryPath();
    String catalogJar = testDir + File.separator + JAR;
    TPCCProjectBuilder pb = new TPCCProjectBuilder();
    pb.addDefaultSchema();
    pb.addDefaultPartitioning();
    pb.addProcedures(MultiSiteSelect.class, InsertNewOrder.class);
    pb.compile(catalogJar, 2, 0);
    // load a catalog
    byte[] bytes = MiscUtils.fileToBytes(new File(catalogJar));
    String serializedCatalog = CatalogUtil.getSerializedCatalogStringFromJar(CatalogUtil.loadAndUpgradeCatalogFromJar(bytes, false).getFirst());
    // create the catalog (that will be passed to the ClientInterface
    catalog = new Catalog();
    catalog.execute(serializedCatalog);
    // update the catalog with the data from the deployment file
    String pathToDeployment = pb.getPathToDeployment();
    assertTrue(CatalogUtil.compileDeployment(catalog, pathToDeployment, false) == null);
    cluster = catalog.getClusters().get("cluster");
    CatalogMap<Procedure> procedures = cluster.getDatabases().get("database").getProcedures();
    Procedure insertProc = procedures.get("InsertNewOrder");
    assert (insertProc != null);
    selectProc = procedures.get("MultiSiteSelect");
    assert (selectProc != null);
    // Each EE needs its own thread for correct initialization.
    final AtomicReference<ExecutionEngine> site1Reference = new AtomicReference<ExecutionEngine>();
    final byte[] configBytes = LegacyHashinator.getConfigureBytes(2);
    Thread site1Thread = new Thread() {

        @Override
        public void run() {
            site1Reference.set(new ExecutionEngineJNI(cluster.getRelativeIndex(), 1, 0, 0, "", 0, 64 * 1024, 100, new HashinatorConfig(HashinatorType.LEGACY, configBytes, 0, 0), false));
        }
    };
    site1Thread.start();
    site1Thread.join();
    final AtomicReference<ExecutionEngine> site2Reference = new AtomicReference<ExecutionEngine>();
    Thread site2Thread = new Thread() {

        @Override
        public void run() {
            site2Reference.set(new ExecutionEngineJNI(cluster.getRelativeIndex(), 2, 1, 0, "", 0, 64 * 1024, 100, new HashinatorConfig(HashinatorType.LEGACY, configBytes, 0, 0), false));
        }
    };
    site2Thread.start();
    site2Thread.join();
    // create two EEs
    ee1 = site1Reference.get();
    ee1.loadCatalog(0, catalog.serialize());
    ee2 = site2Reference.get();
    ee2.loadCatalog(0, catalog.serialize());
    // cache some plan fragments
    selectStmt = selectProc.getStatements().get("selectAll");
    assert (selectStmt != null);
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
        if (i == 0)
            selectTopFrag = f;
        else
            selectBottomFrag = f;
        i++;
    }
    assert (selectTopFrag != null);
    assert (selectBottomFrag != null);
    if (selectTopFrag.getHasdependencies() == false) {
        PlanFragment temp = selectTopFrag;
        selectTopFrag = selectBottomFrag;
        selectBottomFrag = temp;
    }
    // get the insert frag
    Statement insertStmt = insertProc.getStatements().get("insert");
    assert (insertStmt != null);
    for (PlanFragment f : insertStmt.getFragments()) insertFrag = f;
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectTopFrag), Encoder.decodeBase64AndDecompressToBytes(selectTopFrag.getPlannodetree()), selectStmt.getSqltext());
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(insertFrag), Encoder.decodeBase64AndDecompressToBytes(insertFrag.getPlannodetree()), insertStmt.getSqltext());
    // insert some data
    ParameterSet params = ParameterSet.fromArrayNoCopy(1L, 1L, 1L);
    FastDeserializer fragResult2 = ee2.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(insertFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 1, 1, 0, 42, Long.MAX_VALUE, false);
    // ignore totalsize field in message
    fragResult2.readInt();
    VoltTable[] results = TableHelper.convertBackedBufferToTables(fragResult2.buffer(), 1);
    assert (results[0].asScalarLong() == 1L);
    params = ParameterSet.fromArrayNoCopy(2L, 2L, 2L);
    FastDeserializer fragResult1 = ee1.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(insertFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 2, 2, 1, 42, Long.MAX_VALUE, false);
    // ignore totalsize field in message
    fragResult1.readInt();
    results = TableHelper.convertBackedBufferToTables(fragResult1.buffer(), 1);
    assert (fragResult1.buffer() != fragResult2.buffer());
    assert (results[0].asScalarLong() == 1L);
}

Example 8 with PlanFragment

use of org.voltdb.catalog.PlanFragment in project voltdb by VoltDB.

the class TestFragmentProgressUpdate method testTwoUpdates.

@SuppressWarnings("deprecation")
public void testTwoUpdates() throws Exception {
    m_ee.loadCatalog(0, m_catalog.serialize());
    int tableSize = 10000;
    int longOpthreshold = 10000;
    m_warehousedata.clearRowData();
    for (int i = 0; i < tableSize; ++i) {
        m_warehousedata.addRow(i, "name" + i, "st1", "st2", "city", "ST", "zip", 0, 0);
    }
    m_ee.loadTable(WAREHOUSE_TABLEID, m_warehousedata, 0, 0, 0, 0, false, false, Long.MAX_VALUE);
    assertEquals(tableSize, m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    System.out.println("Rows loaded to table " + m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    Statement selectStmt = m_testProc.getStatements().getIgnoreCase("warehouse_select");
    PlanFragment selectBottomFrag = null;
    // delete 5000 records
    // I have no idea what's going on here, and just copy code from the above methods
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
        if (i != 0)
            selectBottomFrag = f;
        i++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    ParameterSet params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    // Like many fully successful operations, a single row fetch counts as 2 logical row operations,
    // one for locating the row and one for retrieving it.
    assertEquals(2, m_ee.m_callsFromEE);
    assertEquals(longOpthreshold * m_ee.m_callsFromEE, m_ee.m_lastTuplesAccessed);
    assertTrue(900000 < m_ee.m_currMemoryInBytes);
    assertTrue(1100000 > m_ee.m_currMemoryInBytes);
    assertTrue(900000 < m_ee.m_peakMemoryInBytes);
    assertTrue(1100000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    long previousPeakMemory = m_ee.m_peakMemoryInBytes;
    Statement deleteStmt = m_testProc.getStatements().getIgnoreCase("warehouse_del_half");
    assertNotNull(deleteStmt);
    PlanFragment deleteBottomFrag = null;
    int j = 0;
    // this kinda assumes the right order
    for (PlanFragment f : deleteStmt.getFragments()) {
        if (j != 0)
            deleteBottomFrag = f;
        j++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(deleteBottomFrag), Encoder.decodeBase64AndDecompressToBytes(deleteBottomFrag.getPlannodetree()), deleteStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(deleteBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, WRITE_TOKEN, false);
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    assertTrue(m_ee.m_callsFromEE > 2);
    // here the m_lastTuplesAccessed is just the same as threshold, since we start a new fragment
    assertEquals(longOpthreshold, m_ee.m_lastTuplesAccessed);
    assertTrue(450000 < m_ee.m_currMemoryInBytes);
    assertTrue(550000 > m_ee.m_currMemoryInBytes);
    assertTrue(450000 < m_ee.m_peakMemoryInBytes);
    assertTrue(550000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    // Although it is true, but I don't think we should compare the memory usage here.
    //assertTrue(m_ee.m_currMemoryInBytes < previousMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes < previousPeakMemory);
}