Examples with Procedure org.voltdb.catalog.Procedure used on opensource projects

Example 6 with Procedure

use of org.voltdb.catalog.Procedure in project voltdb by VoltDB.

the class LoadSinglepartitionTable method run.

/**
     * These parameters, with the exception of ctx, map to user provided values.
     *
     * @param ctx Internal API provided to all system procedures.
     * @param partitionParam Partitioning parameter used to match invocation to partition.
     * @param tableName Name of persistent, parititoned table receiving data.
     * @param table A VoltTable with schema matching the target table containing data to load.
     *              It's assumed that each row in this table partitions to the same partition
     *              as the other rows, and to the same partition as the partition parameter.
     * @param upsertMode True if using upsert instead of insert. If using insert, this proc
     *              will fail if there are any uniqueness constraints violated.
     * @return The number of rows modified. This will be inserts in insert mode, but in upsert
     *              mode, this will be the sum of inserts and updates.
     * @throws VoltAbortException on any failure, but the most common failures are non-matching
     *              partitioning or unique constraint violations.
     */
public long run(SystemProcedureExecutionContext ctx, byte[] partitionParam, String tableName, byte upsertMode, VoltTable table) throws VoltAbortException {
    // if tableName is replicated, fail.
    // otherwise, create a VoltTable for each partition and
    // split up the incoming table .. then send those partial
    // tables to the appropriate sites.
    // Get the metadata object for the table in question from the global metadata/config
    // store, the Catalog.
    Table catTable = ctx.getDatabase().getTables().getIgnoreCase(tableName);
    if (catTable == null) {
        throw new VoltAbortException("Table not present in catalog.");
    }
    // if tableName is replicated, fail.
    if (catTable.getIsreplicated()) {
        throw new VoltAbortException(String.format("LoadSinglepartitionTable incompatible with replicated table %s.", tableName));
    }
    // convert from 8bit signed integer (byte) to boolean
    boolean isUpsert = (upsertMode != 0);
    // upsert requires a primary key on the table to work
    if (isUpsert) {
        boolean hasPkey = false;
        for (Constraint c : catTable.getConstraints()) {
            if (c.getType() == ConstraintType.PRIMARY_KEY.getValue()) {
                hasPkey = true;
                break;
            }
        }
        if (!hasPkey) {
            throw new VoltAbortException(String.format("The --update argument cannot be used for LoadingSinglePartionTable because the table %s does not have a primary key. " + "Either remove the --update argument or add a primary key to the table.", tableName));
        }
    }
    // action should be either "insert" or "upsert"
    final String action = (isUpsert ? "upsert" : "insert");
    // fix any case problems
    tableName = catTable.getTypeName();
    // check that the schema of the input matches
    int columnCount = table.getColumnCount();
    //////////////////////////////////////////////////////////////////////
    // Find the insert/upsert statement for this table
    // This is actually the big trick this procedure does.
    // It borrows the insert plan from the auto-generated insert procedure
    // named "TABLENAME.insert" or "TABLENAME.upsert".
    // We don't like it when users do this stuff, but it is safe in this
    // case.
    //
    // Related code to read is org.voltdb.DefaultProcedureManager, which
    // manages all of the default (CRUD) procedures created lazily for
    // each table in the database, including the plans used here.
    //
    String crudProcName = String.format("%s.%s", tableName, action);
    Procedure p = ctx.ensureDefaultProcLoaded(crudProcName);
    if (p == null) {
        throw new VoltAbortException(String.format("Unable to locate auto-generated CRUD %s statement for table %s", action, tableName));
    }
    // statements of all single-statement procs are named "sql"
    Statement catStmt = p.getStatements().get(VoltDB.ANON_STMT_NAME);
    if (catStmt == null) {
        throw new VoltAbortException(String.format("Unable to find SQL statement for found table %s: BAD", tableName));
    }
    // Create a SQLStmt instance on the fly
    // This unusual to do, as they are typically required to be final instance variables.
    // This only works because the SQL text and plan is identical from the borrowed procedure.
    SQLStmt stmt = new SQLStmt(catStmt.getSqltext());
    m_runner.initSQLStmt(stmt, catStmt);
    long queued = 0;
    long executed = 0;
    // make sure at the start of the table
    table.resetRowPosition();
    // iterate over the rows queueing a sql statement for each row to insert
    for (int i = 0; table.advanceRow(); ++i) {
        Object[] params = new Object[columnCount];
        // get the parameters from the volt table
        for (int col = 0; col < columnCount; ++col) {
            params[col] = table.get(col, table.getColumnType(col));
        }
        // queue an insert and count it
        voltQueueSQL(stmt, params);
        ++queued;
        // 100 is an arbitrary number
        if ((i % 100) == 0) {
            executed += executeSQL();
        }
    }
    // execute any leftover batched statements
    if (queued > executed) {
        executed += executeSQL();
    }
    return executed;
}

Example 7 with Procedure

use of org.voltdb.catalog.Procedure in project voltdb by VoltDB.

the class CatalogSchemaTools method toSchema.

/**
     * Convert a catalog into a string containing all DDL statements.
     * @param catalog
     * @param importLines A set of importLines, should not be mutated.
     * @return String of DDL statements.
     */
public static String toSchema(Catalog catalog, Set<String> importLines) {
    StringBuilder sb = new StringBuilder();
    sb.append("-- This file was generated by VoltDB version ");
    sb.append(VoltDB.instance().getVersionString());
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z");
    String time = sdf.format(System.currentTimeMillis());
    sb.append(" on: " + time + ".\n");
    sb.append("-- This file represents the current database schema.\n");
    sb.append("-- Use this file as input to reproduce the current database structure in another database instance.\n");
    sb.append("--\n");
    sb.append(batchSpecificComments);
    sb.append("-- If the schema declares Java stored procedures, be sure to load the .jar file\n");
    sb.append("-- with the classes before loading the schema. For example:\n");
    sb.append("--\n");
    sb.append("-- LOAD CLASSES voltdb-procs.jar;\n");
    sb.append("-- FILE ddl.sql;\n");
    for (Cluster cluster : catalog.getClusters()) {
        for (Database db : cluster.getDatabases()) {
            toSchema(sb, importLines);
            for (Group grp : db.getGroups()) {
                toSchema(sb, grp);
            }
            sb.append("\n");
            List<Table> viewList = new ArrayList<>();
            CatalogMap<Table> tables = db.getTables();
            if (!tables.isEmpty()) {
                sb.append(startBatch);
                for (Table table : tables) {
                    Object annotation = table.getAnnotation();
                    if (annotation != null && ((TableAnnotation) annotation).ddl != null && table.getMaterializer() != null) {
                        viewList.add(table);
                        continue;
                    }
                    toSchema(sb, table, null, CatalogUtil.isTableExportOnly(db, table), (table.getPartitioncolumn() != null ? table.getPartitioncolumn().getName() : null), CatalogUtil.getExportTargetIfExportTableOrNullOtherwise(db, table));
                }
                // A View cannot precede a table that it depends on in the DDL
                for (Table table : viewList) {
                    String viewQuery = ((TableAnnotation) table.getAnnotation()).ddl;
                    toSchema(sb, table, viewQuery, false, null, null);
                }
            }
            CatalogMap<Procedure> procedures = db.getProcedures();
            if (!procedures.isEmpty()) {
                for (Procedure proc : procedures) {
                    toSchema(sb, proc);
                }
            }
            CatalogMap<Function> functions = db.getFunctions();
            if (!functions.isEmpty()) {
                for (Function func : functions) {
                    toSchema(sb, func);
                }
            }
            if (!tables.isEmpty()) {
                sb.append(endBatch);
            }
        }
    }
    if (dumpSchema) {
        String ts = new SimpleDateFormat("MMddHHmmssSSS").format(new Date());
        File f = new File(String.format("/tmp/canonical-%s.sql", ts));
        try {
            FileWriter fw = new FileWriter(f);
            fw.write(sb.toString());
            fw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    return sb.toString();
}

Example 8 with Procedure

use of org.voltdb.catalog.Procedure in project voltdb by VoltDB.

the class RealVoltDB method printDiagnosticInformation.

public static void printDiagnosticInformation(CatalogContext context, String procName, LoadedProcedureSet procSet) {
    StringBuilder sb = new StringBuilder();
    final CatalogMap<Procedure> catalogProcedures = context.database.getProcedures();
    PureJavaCrc32C crc = new PureJavaCrc32C();
    sb.append("Statements within " + procName + ": ").append("\n");
    for (final Procedure proc : catalogProcedures) {
        if (proc.getTypeName().equals(procName)) {
            for (Statement stmt : proc.getStatements()) {
                // compute hash for determinism check
                crc.reset();
                String sqlText = stmt.getSqltext();
                crc.update(sqlText.getBytes(Constants.UTF8ENCODING));
                int hash = (int) crc.getValue();
                sb.append("Statement Hash: ").append(hash);
                sb.append(", Statement SQL: ").append(sqlText);
                for (PlanFragment frag : stmt.getFragments()) {
                    byte[] planHash = Encoder.hexDecode(frag.getPlanhash());
                    long planId = ActivePlanRepository.getFragmentIdForPlanHash(planHash);
                    String stmtText = ActivePlanRepository.getStmtTextForPlanHash(planHash);
                    byte[] jsonPlan = ActivePlanRepository.planForFragmentId(planId);
                    sb.append(", Plan Fragment Id:").append(planId);
                    sb.append(", Plan Stmt Text:").append(stmtText);
                    sb.append(", Json Plan:").append(new String(jsonPlan));
                }
                sb.append("\n");
            }
        }
    }
    sb.append("Default CRUD Procedures: ").append("\n");
    for (Entry<String, Procedure> pair : context.m_defaultProcs.m_defaultProcMap.entrySet()) {
        crc.reset();
        String sqlText = DefaultProcedureManager.sqlForDefaultProc(pair.getValue());
        crc.update(sqlText.getBytes(Constants.UTF8ENCODING));
        int hash = (int) crc.getValue();
        sb.append("Statement Hash: ").append(hash);
        sb.append(", Statement SQL: ").append(sqlText);
        ProcedureRunner runner = procSet.getProcByName(pair.getValue().getTypeName());
        for (Statement stmt : runner.getCatalogProcedure().getStatements()) {
            for (PlanFragment frag : stmt.getFragments()) {
                byte[] planHash = Encoder.hexDecode(frag.getPlanhash());
                long planId = ActivePlanRepository.getFragmentIdForPlanHash(planHash);
                String stmtText = ActivePlanRepository.getStmtTextForPlanHash(planHash);
                byte[] jsonPlan = ActivePlanRepository.planForFragmentId(planId);
                sb.append(", Plan Fragment Id:").append(planId);
                sb.append(", Plan Stmt Text:").append(stmtText);
                sb.append(", Json Plan:").append(new String(jsonPlan));
            }
        }
        sb.append("\n");
    }
    hostLog.error(sb.toString());
}

Example 9 with Procedure

use of org.voltdb.catalog.Procedure in project voltdb by VoltDB.

the class LoadedProcedureSet method loadUserProcedureRunners.

private static ImmutableMap<String, ProcedureRunner> loadUserProcedureRunners(CatalogContext catalogContext, SiteProcedureConnection site, CatalogSpecificPlanner csp) {
    ImmutableMap.Builder<String, ProcedureRunner> builder = ImmutableMap.<String, ProcedureRunner>builder();
    // load up all the stored procedures
    final CatalogMap<Procedure> catalogProcedures = catalogContext.database.getProcedures();
    for (final Procedure proc : catalogProcedures) {
        // sysprocs found in old catalog versions. (PRO-365)
        if (proc.getTypeName().startsWith("@")) {
            continue;
        }
        // skip non-transactional procs. Those will be handled by LoadedNTProcedureSet
        if (proc.getTransactional() == false) {
            continue;
        }
        VoltProcedure procedure = null;
        if (proc.getHasjava()) {
            final String className = proc.getClassname();
            Class<?> procClass = null;
            try {
                procClass = catalogContext.classForProcedure(className);
            } catch (final ClassNotFoundException e) {
                if (className.startsWith("org.voltdb.")) {
                    String msg = String.format(LoadedProcedureSet.ORGVOLTDB_PROCNAME_ERROR_FMT, className);
                    VoltDB.crashLocalVoltDB(msg, false, null);
                } else {
                    String msg = String.format(LoadedProcedureSet.UNABLETOLOAD_ERROR_FMT, className);
                    VoltDB.crashLocalVoltDB(msg, false, null);
                }
            }
            try {
                procedure = (VoltProcedure) procClass.newInstance();
            } catch (final Exception e) {
                // TODO: remove the extra meaningless parameter "0"
                hostLog.l7dlog(Level.WARN, LogKeys.host_ExecutionSite_GenericException.name(), new Object[] { site.getCorrespondingSiteId(), 0 }, e);
            }
        } else {
            procedure = new ProcedureRunner.StmtProcedure();
        }
        assert (procedure != null);
        ProcedureRunner runner = new ProcedureRunner(procedure, site, proc, csp);
        builder.put(proc.getTypeName().intern(), runner);
    }
    return builder.build();
}

Example 10 with Procedure

use of org.voltdb.catalog.Procedure in project voltdb by VoltDB.

the class VoltCompiler method compileProcedures.

/**
     * @param db the database entry in the catalog
     * @param hsql an interface to the hsql frontend, initialized and potentially reused by the caller.
     * @param classDependencies
     * @param voltDdlTracker non-standard VoltDB schema annotations
     * @param whichProcs indicates which ddl-defined procedures to load: none, single-statement, or all
     * @throws VoltCompilerException
     */
private void compileProcedures(Database db, HSQLInterface hsql, Collection<ProcedureDescriptor> allProcs, Collection<Class<?>> classDependencies, DdlProceduresToLoad whichProcs, CatalogMap<Procedure> prevProcsIfAny, InMemoryJarfile jarOutput) throws VoltCompilerException {
    // build a cache of previous SQL stmts
    m_previousCatalogStmts.clear();
    if (prevProcsIfAny != null) {
        for (Procedure prevProc : prevProcsIfAny) {
            for (Statement prevStmt : prevProc.getStatements()) {
                addStatementToCache(prevStmt);
            }
        }
    }
    // dependencies for its "dry run" on an unchanged application ddl file.
    if (whichProcs == DdlProceduresToLoad.ALL_DDL_PROCEDURES) {
        // Add all the class dependencies to the output jar
        for (final Class<?> classDependency : classDependencies) {
            addClassToJar(jarOutput, classDependency);
        }
    }
    final List<ProcedureDescriptor> procedures = new ArrayList<>();
    procedures.addAll(allProcs);
    // Actually parse and handle all the Procedures
    for (final ProcedureDescriptor procedureDescriptor : procedures) {
        final String procedureName = procedureDescriptor.m_className;
        if (procedureDescriptor.m_singleStmt == null) {
            m_currentFilename = procedureName.substring(procedureName.lastIndexOf('.') + 1);
            m_currentFilename += ".class";
        } else if (whichProcs == DdlProceduresToLoad.ONLY_SINGLE_STATEMENT_PROCEDURES) {
            // for all the single-statement procedures of an unchanged application ddl file.
            continue;
        } else {
            m_currentFilename = procedureName;
        }
        ProcedureCompiler.compile(this, hsql, m_estimates, db, procedureDescriptor, jarOutput);
    }
    // done handling files
    m_currentFilename = NO_FILENAME;
    // allow gc to reclaim any cache memory here
    m_previousCatalogStmts.clear();
}