Examples with Statement org.voltdb.catalog.Statement used on opensource projects

Example 11 with Statement

use of org.voltdb.catalog.Statement in project voltdb by VoltDB.

the class LoadMultipartitionTable method run.

/**
     * These parameters, with the exception of ctx, map to user provided values.
     *
     * @param ctx
     *            Internal. Not a user-supplied parameter.
     * @param tableName
     *            Name of persistent table receiving data.
     * @param table
     *            A VoltTable with schema matching tableName containing data to
     *            load.
     * @param upsertMode
     *            True if using upsert instead of insert
     * @return {@link org.voltdb.VoltSystemProcedure#STATUS_SCHEMA}
     * @throws VoltAbortException
     */
public long run(SystemProcedureExecutionContext ctx, String tableName, byte upsertMode, VoltTable table) throws VoltAbortException {
    // if tableName is replicated, just send table everywhere.
    // otherwise, create a VoltTable for each partition and
    // split up the incoming table .. then send those partial
    // tables to the appropriate sites.
    Table catTable = ctx.getDatabase().getTables().getIgnoreCase(tableName);
    if (catTable == null) {
        throw new VoltAbortException("Table not present in catalog.");
    }
    boolean isUpsert = (upsertMode != 0);
    if (isUpsert) {
        boolean hasPkey = false;
        for (Constraint c : catTable.getConstraints()) {
            if (c.getType() == ConstraintType.PRIMARY_KEY.getValue()) {
                hasPkey = true;
                break;
            }
        }
        if (!hasPkey) {
            throw new VoltAbortException(String.format("The --update argument cannot be used for LoadMultipartitionTable because the table %s does not have a primary key. " + "Either remove the --update argument or add a primary key to the table.", tableName));
        }
    }
    // action should be either "insert" or "upsert"
    final String action = (isUpsert ? "upsert" : "insert");
    // fix any case problems
    tableName = catTable.getTypeName();
    // check that the schema of the input matches
    int columnCount = table.getColumnCount();
    // find the insert statement for this table
    String crudProcName = String.format("%s.%s", tableName.toUpperCase(), action);
    Procedure proc = ctx.ensureDefaultProcLoaded(crudProcName);
    if (proc == null) {
        throw new VoltAbortException(String.format("Unable to locate auto-generated CRUD %s statement for table %s", action, tableName));
    }
    // ensure MP fragment tasks load the plan for the table loading procedure
    m_runner.setProcNameToLoadForFragmentTasks(crudProcName);
    Statement catStmt = proc.getStatements().get(VoltDB.ANON_STMT_NAME);
    if (catStmt == null) {
        throw new VoltAbortException(String.format("Unable to find SQL statement for found table %s: BAD", tableName));
    }
    // create a SQLStmt instance on the fly (unusual to do)
    SQLStmt stmt = new SQLStmt(catStmt.getSqltext());
    m_runner.initSQLStmt(stmt, catStmt);
    if (catTable.getIsreplicated()) {
        long queued = 0;
        long executed = 0;
        // make sure at the start of the table
        table.resetRowPosition();
        for (int i = 0; table.advanceRow(); ++i) {
            Object[] params = new Object[columnCount];
            // get the parameters from the volt table
            for (int col = 0; col < columnCount; ++col) {
                params[col] = table.get(col, table.getColumnType(col));
            }
            // queue an insert and count it
            voltQueueSQL(stmt, params);
            ++queued;
            // 100 is an arbitrary number
            if ((i % 100) == 0) {
                executed += executeSQL();
            }
        }
        // execute any leftover batched statements
        if (queued > executed) {
            executed += executeSQL();
        }
        return executed;
    } else {
        throw new VoltAbortException("LoadMultipartitionTable no longer supports loading partitioned tables" + " use CRUD procs instead");
    }
}

Example 12 with Statement

use of org.voltdb.catalog.Statement in project voltdb by VoltDB.

the class PlannerTestAideDeCamp method compile.

/**
     * Compile and cache the statement and plan and return the final plan graph.
     */
private List<AbstractPlanNode> compile(String sql, int paramCount, String joinOrder, boolean inferPartitioning, boolean forceSingle, DeterminismMode detMode) {
    String stmtLabel = "stmt-" + String.valueOf(compileCounter++);
    Statement catalogStmt = proc.getStatements().add(stmtLabel);
    catalogStmt.setSqltext(sql);
    catalogStmt.setSinglepartition(forceSingle);
    // determine the type of the query
    QueryType qtype = QueryType.SELECT;
    catalogStmt.setReadonly(true);
    if (sql.toLowerCase().startsWith("insert")) {
        qtype = QueryType.INSERT;
        catalogStmt.setReadonly(false);
    }
    if (sql.toLowerCase().startsWith("update")) {
        qtype = QueryType.UPDATE;
        catalogStmt.setReadonly(false);
    }
    if (sql.toLowerCase().startsWith("delete")) {
        qtype = QueryType.DELETE;
        catalogStmt.setReadonly(false);
    }
    catalogStmt.setQuerytype(qtype.getValue());
    // name will look like "basename-stmt-#"
    String name = catalogStmt.getParent().getTypeName() + "-" + catalogStmt.getTypeName();
    DatabaseEstimates estimates = new DatabaseEstimates();
    TrivialCostModel costModel = new TrivialCostModel();
    StatementPartitioning partitioning;
    if (inferPartitioning) {
        partitioning = StatementPartitioning.inferPartitioning();
    } else if (forceSingle) {
        partitioning = StatementPartitioning.forceSP();
    } else {
        partitioning = StatementPartitioning.forceMP();
    }
    String procName = catalogStmt.getParent().getTypeName();
    QueryPlanner planner = new QueryPlanner(sql, stmtLabel, procName, db, partitioning, hsql, estimates, false, StatementCompiler.DEFAULT_MAX_JOIN_TABLES, costModel, null, joinOrder, detMode);
    CompiledPlan plan = null;
    planner.parse();
    plan = planner.plan();
    assert (plan != null);
    // Partitioning optionally inferred from the planning process.
    if (partitioning.isInferred()) {
        catalogStmt.setSinglepartition(partitioning.isInferredSingle());
    }
    // We will need to update the system catalogs with this new information
    for (int i = 0; i < plan.parameters.length; ++i) {
        StmtParameter catalogParam = catalogStmt.getParameters().add(String.valueOf(i));
        ParameterValueExpression pve = plan.parameters[i];
        catalogParam.setJavatype(pve.getValueType().getValue());
        catalogParam.setIsarray(pve.getParamIsVector());
        catalogParam.setIndex(i);
    }
    List<PlanNodeList> nodeLists = new ArrayList<>();
    nodeLists.add(new PlanNodeList(plan.rootPlanGraph));
    if (plan.subPlanGraph != null) {
        nodeLists.add(new PlanNodeList(plan.subPlanGraph));
    }
    // Now update our catalog information
    // HACK: We're using the node_tree's hashCode() as it's name. It would be really
    //     nice if the Catalog code give us an guid without needing a name first...
    String json = null;
    try {
        JSONObject jobj = new JSONObject(nodeLists.get(0).toJSONString());
        json = jobj.toString(4);
    } catch (JSONException e2) {
        // TODO Auto-generated catch block
        e2.printStackTrace();
        System.exit(-1);
        return null;
    }
    //
    try {
        BuildDirectoryUtils.writeFile("statement-plans", name + "_json.txt", json, true);
        BuildDirectoryUtils.writeFile("statement-plans", name + ".dot", nodeLists.get(0).toDOTString("name"), true);
    } catch (Exception e) {
        e.printStackTrace();
    }
    List<AbstractPlanNode> plannodes = new ArrayList<>();
    for (PlanNodeList nodeList : nodeLists) {
        plannodes.add(nodeList.getRootPlanNode());
    }
    m_currentPlan = plan;
    return plannodes;
}

Example 13 with Statement

use of org.voltdb.catalog.Statement in project voltdb by VoltDB.

the class TestFragmentProgressUpdate method testTwoUpdates.

@SuppressWarnings("deprecation")
public void testTwoUpdates() throws Exception {
    m_ee.loadCatalog(0, m_catalog.serialize());
    int tableSize = 10000;
    int longOpthreshold = 10000;
    m_warehousedata.clearRowData();
    for (int i = 0; i < tableSize; ++i) {
        m_warehousedata.addRow(i, "name" + i, "st1", "st2", "city", "ST", "zip", 0, 0);
    }
    m_ee.loadTable(WAREHOUSE_TABLEID, m_warehousedata, 0, 0, 0, 0, false, false, Long.MAX_VALUE);
    assertEquals(tableSize, m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    System.out.println("Rows loaded to table " + m_ee.serializeTable(WAREHOUSE_TABLEID).getRowCount());
    Statement selectStmt = m_testProc.getStatements().getIgnoreCase("warehouse_select");
    PlanFragment selectBottomFrag = null;
    // delete 5000 records
    // I have no idea what's going on here, and just copy code from the above methods
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
        if (i != 0)
            selectBottomFrag = f;
        i++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    ParameterSet params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    // Like many fully successful operations, a single row fetch counts as 2 logical row operations,
    // one for locating the row and one for retrieving it.
    assertEquals(2, m_ee.m_callsFromEE);
    assertEquals(longOpthreshold * m_ee.m_callsFromEE, m_ee.m_lastTuplesAccessed);
    assertTrue(900000 < m_ee.m_currMemoryInBytes);
    assertTrue(1100000 > m_ee.m_currMemoryInBytes);
    assertTrue(900000 < m_ee.m_peakMemoryInBytes);
    assertTrue(1100000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    long previousPeakMemory = m_ee.m_peakMemoryInBytes;
    Statement deleteStmt = m_testProc.getStatements().getIgnoreCase("warehouse_del_half");
    assertNotNull(deleteStmt);
    PlanFragment deleteBottomFrag = null;
    int j = 0;
    // this kinda assumes the right order
    for (PlanFragment f : deleteStmt.getFragments()) {
        if (j != 0)
            deleteBottomFrag = f;
        j++;
    }
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(deleteBottomFrag), Encoder.decodeBase64AndDecompressToBytes(deleteBottomFrag.getPlannodetree()), deleteStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(deleteBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, WRITE_TOKEN, false);
    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(CatalogUtil.getUniqueIdForFragment(selectBottomFrag), Encoder.decodeBase64AndDecompressToBytes(selectBottomFrag.getPlannodetree()), selectStmt.getSqltext());
    params = ParameterSet.emptyParameterSet();
    m_ee.executePlanFragments(1, new long[] { CatalogUtil.getUniqueIdForFragment(selectBottomFrag) }, null, new ParameterSet[] { params }, null, new String[] { selectStmt.getSqltext() }, null, null, 3, 3, 2, 42, Long.MAX_VALUE, false);
    assertTrue(m_ee.m_callsFromEE > 2);
    // here the m_lastTuplesAccessed is just the same as threshold, since we start a new fragment
    assertEquals(longOpthreshold, m_ee.m_lastTuplesAccessed);
    assertTrue(450000 < m_ee.m_currMemoryInBytes);
    assertTrue(550000 > m_ee.m_currMemoryInBytes);
    assertTrue(450000 < m_ee.m_peakMemoryInBytes);
    assertTrue(550000 > m_ee.m_peakMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes >= m_ee.m_currMemoryInBytes);
    // Although it is true, but I don't think we should compare the memory usage here.
    //assertTrue(m_ee.m_currMemoryInBytes < previousMemoryInBytes);
    assertTrue(m_ee.m_peakMemoryInBytes < previousPeakMemory);
}

Example 14 with Statement

use of org.voltdb.catalog.Statement in project voltdb by VoltDB.

the class ViewExplainer method explain.

public static ArrayList<String[]> explain(Table viewTable) throws Exception {
    String viewName = viewTable.getTypeName();
    MaterializedViewHandlerInfo mvHandlerInfo = viewTable.getMvhandlerinfo().get("mvHandlerInfo");
    MaterializedViewInfo mvInfo = null;
    CatalogMap<Statement> fallBackQueryStmts;
    List<Column> destColumnArray = CatalogUtil.getSortedCatalogItems(viewTable.getColumns(), "index");
    ArrayList<String[]> retval = new ArrayList<String[]>();
    // Is this view single-table?
    if (mvHandlerInfo == null) {
        // (Legacy code for single table view uses a different model and code path)
        if (viewTable.getMaterializer() == null) {
            // If we cannot find view metadata from both locations, this table is not a materialized view.
            throw new Exception("Table " + viewName + " is not a view.");
        }
        mvInfo = viewTable.getMaterializer().getViews().get(viewName);
        fallBackQueryStmts = mvInfo.getFallbackquerystmts();
    } else {
        // For multi-table views we need to show the query plan for evaluating joins.
        Statement createQuery = mvHandlerInfo.getCreatequery().get("createQuery");
        retval.add(new String[] { "Join Evaluation", Encoder.hexDecodeToString(createQuery.getExplainplan()) });
        fallBackQueryStmts = mvHandlerInfo.getFallbackquerystmts();
    }
    // For each min/max column find out if an execution plan is used.
    int minMaxAggIdx = 0;
    for (int j = 0; j < destColumnArray.size(); j++) {
        Column destColumn = destColumnArray.get(j);
        ExpressionType aggType = ExpressionType.get(destColumn.getAggregatetype());
        if (aggType == ExpressionType.AGGREGATE_MIN || aggType == ExpressionType.AGGREGATE_MAX) {
            Statement fallBackQueryStmt = fallBackQueryStmts.get(String.valueOf(minMaxAggIdx));
            // How this min/max will be refreshed?
            String plan = "";
            //   * execution plan
            if (mvHandlerInfo == null) {
                CatalogMap<IndexRef> hardCodedIndicesForSingleTableView = mvInfo.getIndexforminmax();
                String hardCodedIndexName = hardCodedIndicesForSingleTableView.get(String.valueOf(minMaxAggIdx)).getName();
                String indexNameUsedInStatement = getIndexNameUsedInStatement(fallBackQueryStmt);
                if (!indexNameUsedInStatement.equalsIgnoreCase(hardCodedIndexName)) {
                    plan = Encoder.hexDecodeToString(fallBackQueryStmt.getExplainplan());
                }
                // If we do not use execution plan, see which built-in method is used.
                if (plan.equals("")) {
                    if (hardCodedIndexName.equals("")) {
                        plan = "Built-in sequential scan.";
                    } else {
                        plan = "Built-in index scan \"" + hardCodedIndexName + "\".";
                    }
                }
            } else {
                plan = Encoder.hexDecodeToString(fallBackQueryStmt.getExplainplan());
            }
            retval.add(new String[] { "Refresh " + (aggType == ExpressionType.AGGREGATE_MIN ? "MIN" : "MAX") + " column \"" + destColumn.getName() + "\"", plan });
            minMaxAggIdx++;
        }
    }
    return retval;
}

Example 15 with Statement

use of org.voltdb.catalog.Statement in project voltdb by VoltDB.

the class ReportMaker method generateProcedureRow.

static String generateProcedureRow(CatalogMap<Table> tables, Procedure procedure) {
    StringBuilder sb = new StringBuilder();
    sb.append("<tr class='primaryrow'>");
    // column 1: procedure name
    String anchor = procedure.getTypeName().toLowerCase();
    sb.append("<td style='white-space: nowrap'><i id='p-" + anchor + "--icon' class='icon-chevron-right'></i> <a href='#p-");
    sb.append(anchor).append("' id='p-").append(anchor).append("' class='togglex'>");
    sb.append(procedure.getTypeName());
    sb.append("</a></td>");
    // column 2: parameter types
    sb.append("<td>");
    List<ProcParameter> params = CatalogUtil.getSortedCatalogItems(procedure.getParameters(), "index");
    List<String> paramTypes = new ArrayList<String>();
    for (ProcParameter param : params) {
        String paramType = VoltType.get((byte) param.getType()).name();
        if (param.getIsarray()) {
            paramType += "[]";
        }
        paramTypes.add(paramType);
    }
    if (paramTypes.size() == 0) {
        sb.append("<i>None</i>");
    }
    sb.append(StringUtils.join(paramTypes, ", "));
    sb.append("</td>");
    // column 3: partitioning
    sb.append("<td>");
    if (procedure.getSinglepartition()) {
        tag(sb, "success", "Single");
    } else {
        tag(sb, "warning", "Multi");
    }
    sb.append("</td>");
    // column 4: read/write
    sb.append("<td>");
    if (procedure.getReadonly()) {
        tag(sb, "success", "Read");
    } else {
        tag(sb, "warning", "Write");
    }
    sb.append("</td>");
    // column 5: access
    sb.append("<td>");
    List<String> groupNames = new ArrayList<String>();
    for (GroupRef groupRef : procedure.getAuthgroups()) {
        groupNames.add(groupRef.getGroup().getTypeName());
    }
    if (groupNames.size() == 0) {
        sb.append("<i>None</i>");
    }
    sb.append(StringUtils.join(groupNames, ", "));
    sb.append("</td>");
    // column 6: attributes
    sb.append("<td>");
    if (procedure.getHasjava()) {
        tag(sb, "info", "Java");
    } else {
        tag(sb, null, "Single-Stmt");
    }
    boolean isND = false;
    int scanCount = 0;
    for (Statement stmt : procedure.getStatements()) {
        scanCount += stmt.getSeqscancount();
        if (!stmt.getIscontentdeterministic() || !stmt.getIsorderdeterministic()) {
            isND = false;
        }
    }
    if (isND) {
        tag(sb, "inverse", "Determinism");
    }
    if (scanCount > 0) {
        tag(sb, "important", "Scans");
    }
    sb.append("</td>");
    sb.append("</tr>\n");
    // BUILD THE DROPDOWN FOR THE STATEMENT/DETAIL TABLE
    sb.append("<tr class='tablesorter-childRow'><td class='invert' colspan='6' id='p-" + procedure.getTypeName().toLowerCase() + "--dropdown'>\n");
    // output partitioning parameter info
    if (procedure.getSinglepartition()) {
        String pTable = procedure.getPartitioncolumn().getParent().getTypeName();
        String pColumn = procedure.getPartitioncolumn().getTypeName();
        int pIndex = procedure.getPartitionparameter();
        sb.append(String.format("<p>Partitioned on parameter %d which maps to column %s" + " of table <a class='invert' href='#s-%s'>%s</a>.</p>", pIndex, pColumn, pTable, pTable));
    }
    // get the annotation or ensure it's there
    ProcedureAnnotation annotation = (ProcedureAnnotation) procedure.getAnnotation();
    if (annotation == null) {
        annotation = new ProcedureAnnotation();
        procedure.setAnnotation(annotation);
    }
    // this needs to be run before the ProcedureAnnotation is used below
    // because it modifies it
    String statementsTable = generateStatementsTable(tables, procedure);
    // output what schema this interacts with
    // make sure tables appear in only one category
    annotation.tablesRead.removeAll(annotation.tablesUpdated);
    if (annotation.tablesRead.size() > 0) {
        sb.append("<p>Read-only access to tables: ");
        List<String> tableList = new ArrayList<String>();
        for (Table table : annotation.tablesRead) {
            tableList.add("<a href='#s-" + table.getTypeName() + "'>" + table.getTypeName() + "</a>");
        }
        sb.append(StringUtils.join(tableList, ", "));
        sb.append("</p>");
    }
    if (annotation.tablesUpdated.size() > 0) {
        sb.append("<p>Read/Write access to tables: ");
        List<String> tableList = new ArrayList<String>();
        for (Table table : annotation.tablesUpdated) {
            tableList.add("<a href='#s-" + table.getTypeName() + "'>" + table.getTypeName() + "</a>");
        }
        sb.append(StringUtils.join(tableList, ", "));
        sb.append("</p>");
    }
    if (annotation.indexesUsed.size() > 0) {
        sb.append("<p>Uses indexes: ");
        List<String> indexes = new ArrayList<String>();
        for (Index index : annotation.indexesUsed) {
            Table table = (Table) index.getParent();
            indexes.add("<a href='#s-" + table.getTypeName() + "-" + index.getTypeName() + "'>" + index.getTypeName() + "</a>");
        }
        sb.append(StringUtils.join(indexes, ", "));
        sb.append("</p>");
    }
    sb.append(statementsTable);
    sb.append("</td></tr>\n");
    return sb.toString();
}