Examples with VoltCompilerException org.voltdb.compiler.VoltCompiler.VoltCompilerException used on opensource projects

Example 16 with VoltCompilerException

use of org.voltdb.compiler.VoltCompiler.VoltCompilerException in project voltdb by VoltDB.

the class DDLCompiler method processVoltDBStatements.

private void processVoltDBStatements(final Database db, final DdlProceduresToLoad whichProcs, DDLStatement stmt) throws VoltCompilerException {
    boolean processed = false;
    try {
        // Process a VoltDB-specific DDL statement, like PARTITION, REPLICATE,
        // CREATE PROCEDURE, CREATE FUNCTION, and CREATE ROLE.
        processed = m_voltStatementProcessor.process(stmt, db, whichProcs);
    } catch (VoltCompilerException e) {
        // Reformat the message thrown by VoltDB DDL processing to have a line number.
        String msg = "VoltDB DDL Error: \"" + e.getMessage() + "\" in statement starting on lineno: " + stmt.lineNo;
        throw m_compiler.new VoltCompilerException(msg);
    }
    if (!processed) {
        try {
            //* enable to debug */ System.out.println("DEBUG: " + stmt.statement);
            // kind of ugly.  We hex-encode each statement so we can
            // avoid embedded newlines so we can delimit statements
            // with newline.
            m_fullDDL += Encoder.hexEncode(stmt.statement) + "\n";
            // figure out what table this DDL might affect to minimize diff processing
            HSQLDDLInfo ddlStmtInfo = HSQLLexer.preprocessHSQLDDL(stmt.statement);
            // Get the diff that results from applying this statement and apply it
            // to our local tree (with Volt-specific additions)
            VoltXMLDiff thisStmtDiff = m_hsql.runDDLCommandAndDiff(ddlStmtInfo, stmt.statement);
            // null diff means no change (usually drop if exists for non-existent thing)
            if (thisStmtDiff != null) {
                applyDiff(thisStmtDiff);
            }
            // special treatment for stream syntax
            if (ddlStmtInfo.creatStream) {
                processCreateStreamStatement(stmt, db, whichProcs);
            }
        } catch (HSQLParseException e) {
            String msg = "DDL Error: \"" + e.getMessage() + "\" in statement starting on lineno: " + stmt.lineNo;
            throw m_compiler.new VoltCompilerException(msg, stmt.lineNo);
        }
    }
}

Example 17 with VoltCompilerException

use of org.voltdb.compiler.VoltCompiler.VoltCompilerException in project voltdb by VoltDB.

the class DDLCompiler method buildPartialIndexPredicate.

/**
     * Build the abstract expression representing the partial index predicate.
     * Verify it satisfies the rules. Throw error messages otherwise.
     *
     * @param dummy AbstractParsedStmt
     * @param indexName The name of the index being checked.
     * @param predicateXML The XML representing the predicate.
     * @param table Table
     * @throws VoltCompilerException
     * @return AbstractExpression
     */
private static AbstractExpression buildPartialIndexPredicate(AbstractParsedStmt dummy, String indexName, VoltXMLElement predicateXML, Table table, VoltCompiler compiler) throws VoltCompilerException {
    // Make sure all column expressions refer to the same index table
    // before we can parse the XML to avoid the AbstractParsedStmt
    // exception/assertion
    String tableName = table.getTypeName();
    assert (tableName != null);
    String msg = "Partial index \"" + indexName + "\" ";
    // Make sure all column expressions refer the index table
    List<VoltXMLElement> columnRefs = predicateXML.findChildrenRecursively("columnref");
    for (VoltXMLElement columnRef : columnRefs) {
        String columnRefTableName = columnRef.attributes.get("table");
        if (columnRefTableName != null && !tableName.equals(columnRefTableName)) {
            msg += "with expression(s) involving other tables is not supported.";
            throw compiler.new VoltCompilerException(msg);
        }
    }
    // Now it safe to parse the expression tree
    AbstractExpression predicate = dummy.parseExpressionTree(predicateXML);
    if (predicate.hasAnySubexpressionOfClass(AggregateExpression.class)) {
        msg += "with aggregate expression(s) is not supported.";
        throw compiler.new VoltCompilerException(msg);
    }
    if (predicate.hasAnySubexpressionOfClass(AbstractSubqueryExpression.class)) {
        msg += "with subquery expression(s) is not supported.";
        throw compiler.new VoltCompilerException(msg);
    }
    return predicate;
}

Example 18 with VoltCompilerException

use of org.voltdb.compiler.VoltCompiler.VoltCompilerException in project voltdb by VoltDB.

the class DDLCompiler method loadSchemaWithFiltering.

public void loadSchemaWithFiltering(Reader reader, final Database db, final DdlProceduresToLoad whichProcs, SQLParser.FileInfo fileInfo) throws VoltCompiler.VoltCompilerException {
    final LineReaderAdapter lineReader = new LineReaderAdapter(reader);
    DDLParserCallback callback = new DDLParserCallback() {

        @Override
        public void statement(String statement, int lineNum) throws VoltCompiler.VoltCompilerException {
            try {
                if (statement == null || statement.trim().isEmpty()) {
                    return;
                }
                doExecuteStatement(statement.trim() + ";", lineNum);
            } catch (Exception e) {
                throw m_compiler.new VoltCompilerException(e);
            }
        }

        @Override
        public void batch(String batch, int batchEndLineNumber) throws VoltCompiler.VoltCompilerException {
            try {
                String[] parts = batch.split(";");
                int currLine = batchEndLineNumber - parts.length;
                for (String s : parts) {
                    if (!s.trim().isEmpty()) {
                        doExecuteStatement(s.trim() + ";", currLine++);
                    }
                }
            } catch (Exception e) {
                throw m_compiler.new VoltCompilerException(e);
            }
        }

        private void doExecuteStatement(String statement, int lineNum) throws VoltCompiler.VoltCompilerException {
            // Filter out any statements that we don't want to deal with
            if (statement.toUpperCase().startsWith("LOAD")) {
                return;
            }
            DDLStatement stmt = new DDLStatement(statement, lineNum);
            processVoltDBStatements(db, whichProcs, stmt);
        }
    };
    try {
        SQLCommand.executeScriptFromReader(fileInfo, lineReader, callback);
    } catch (Exception e) {
        throw m_compiler.new VoltCompilerException(e);
    } finally {
        // NEEDSWORK (ENG-12255): This really should be done outside the loop, by the caller. The exception propagation, at this point, is pretty messy, and
        // changing it would be a pretty big effort that's really not relevant to this ticket. Fix this sometime.
        lineReader.close();
    }
    // process extra classes
    m_tracker.addExtraClasses(m_classMatcher.getMatchedClassList());
    // possibly save some memory
    m_classMatcher.clear();
}

Example 19 with VoltCompilerException

use of org.voltdb.compiler.VoltCompiler.VoltCompilerException in project voltdb by VoltDB.

the class DDLCompiler method addTableToCatalog.

private void addTableToCatalog(Database db, VoltXMLElement node, boolean isXDCR) throws VoltCompilerException {
    assert node.name.equals("table");
    // Construct table-specific maps
    HashMap<String, Column> columnMap = new HashMap<>();
    HashMap<String, Index> indexMap = new HashMap<>();
    final String name = node.attributes.get("name");
    // create a table node in the catalog
    final Table table = db.getTables().add(name);
    // set max value before return for view table
    table.setTuplelimit(Integer.MAX_VALUE);
    // add the original DDL to the table (or null if it's not there)
    TableAnnotation annotation = new TableAnnotation();
    table.setAnnotation(annotation);
    // handle the case where this is a materialized view
    final String query = node.attributes.get("query");
    if (query != null) {
        assert (query.length() > 0);
        m_matViewMap.put(table, query);
    }
    final boolean isStream = (node.attributes.get("stream") != null);
    final String streamTarget = node.attributes.get("export");
    final String streamPartitionColumn = node.attributes.get("partitioncolumn");
    // all tables start replicated
    // if a partition is found in the project file later,
    //  then this is reversed
    table.setIsreplicated(true);
    // map of index replacements for later constraint fixup
    final Map<String, String> indexReplacementMap = new TreeMap<>();
    // Need the columnTypes sorted by column index.
    SortedMap<Integer, VoltType> columnTypes = new TreeMap<>();
    for (VoltXMLElement subNode : node.children) {
        if (subNode.name.equals("columns")) {
            int colIndex = 0;
            for (VoltXMLElement columnNode : subNode.children) {
                if (columnNode.name.equals("column")) {
                    addColumnToCatalog(table, columnNode, columnTypes, columnMap, m_compiler);
                    colIndex++;
                }
            }
            // limit the total number of columns in a table
            if (colIndex > MAX_COLUMNS) {
                String msg = "Table " + name + " has " + colIndex + " columns (max is " + MAX_COLUMNS + ")";
                throw m_compiler.new VoltCompilerException(msg);
            }
        }
        if (subNode.name.equals("indexes")) {
            // that refer to them.
            for (VoltXMLElement indexNode : subNode.children) {
                if (indexNode.name.equals("index") == false)
                    continue;
                String indexName = indexNode.attributes.get("name");
                if (indexName.startsWith(HSQLInterface.AUTO_GEN_IDX_PREFIX) == false) {
                    addIndexToCatalog(db, table, indexNode, indexReplacementMap, indexMap, columnMap, m_compiler);
                }
            }
            for (VoltXMLElement indexNode : subNode.children) {
                if (indexNode.name.equals("index") == false)
                    continue;
                String indexName = indexNode.attributes.get("name");
                if (indexName.startsWith(HSQLInterface.AUTO_GEN_IDX_PREFIX) == true) {
                    addIndexToCatalog(db, table, indexNode, indexReplacementMap, indexMap, columnMap, m_compiler);
                }
            }
        }
        if (subNode.name.equals("constraints")) {
            for (VoltXMLElement constraintNode : subNode.children) {
                if (constraintNode.name.equals("constraint")) {
                    addConstraintToCatalog(table, constraintNode, indexReplacementMap, indexMap);
                }
            }
        }
    }
    // Warn user if DR table don't have any unique index.
    if (isXDCR && node.attributes.get("drTable") != null && node.attributes.get("drTable").equalsIgnoreCase("ENABLE")) {
        boolean hasUniqueIndex = false;
        for (Index index : table.getIndexes()) {
            if (index.getUnique()) {
                hasUniqueIndex = true;
                break;
            }
        }
        if (!hasUniqueIndex) {
            String info = String.format("Table %s doesn't have any unique index, it will cause full table scans to update/delete DR record and may become slower as table grow.", table.getTypeName());
            m_compiler.addWarn(info);
        }
    }
    table.setSignature(CatalogUtil.getSignatureForTable(name, columnTypes));
    /*
         * Validate that each variable-length column is below the max value length,
         * and that the maximum size for the row is below the max row length.
         */
    int maxRowSize = 0;
    for (Column c : columnMap.values()) {
        VoltType t = VoltType.get((byte) c.getType());
        if (t == VoltType.STRING && (!c.getInbytes())) {
            if (c.getSize() * MAX_BYTES_PER_UTF8_CHARACTER > VoltType.MAX_VALUE_LENGTH) {
                throw m_compiler.new VoltCompilerException("Column " + name + "." + c.getName() + " specifies a maximum size of " + c.getSize() + " characters" + " but the maximum supported size is " + VoltType.humanReadableSize(VoltType.MAX_VALUE_LENGTH / MAX_BYTES_PER_UTF8_CHARACTER) + " characters or " + VoltType.humanReadableSize(VoltType.MAX_VALUE_LENGTH) + " bytes");
            }
            maxRowSize += 4 + c.getSize() * MAX_BYTES_PER_UTF8_CHARACTER;
        } else if (t.isVariableLength()) {
            if (c.getSize() > VoltType.MAX_VALUE_LENGTH) {
                throw m_compiler.new VoltCompilerException("Column " + name + "." + c.getName() + " specifies a maximum size of " + c.getSize() + " bytes" + " but the maximum supported size is " + VoltType.humanReadableSize(VoltType.MAX_VALUE_LENGTH));
            }
            maxRowSize += 4 + c.getSize();
        } else {
            maxRowSize += t.getLengthInBytesForFixedTypes();
        }
    }
    if (maxRowSize > MAX_ROW_SIZE) {
        throw m_compiler.new VoltCompilerException("Error: Table " + name + " has a maximum row size of " + maxRowSize + " but the maximum supported row size is " + MAX_ROW_SIZE);
    }
    // the DDL statement for the VIEW
    if (query != null) {
        annotation.ddl = query;
    } else {
        // Get the final DDL for the table rebuilt from the catalog object
        // Don't need a real StringBuilder or export state to get the CREATE for a table
        annotation.ddl = CatalogSchemaTools.toSchema(new StringBuilder(), table, query, isStream, streamPartitionColumn, streamTarget);
    }
}

Example 20 with VoltCompilerException

use of org.voltdb.compiler.VoltCompiler.VoltCompilerException in project voltdb by VoltDB.

the class UpdateApplicationBase method prepareApplicationCatalogDiff.

/**
     *
     * @param operationBytes The bytes for the catalog operation, if any. May be null in all cases.
     * For UpdateApplicationCatalog, this will contain the compiled catalog jarfile bytes
     * For UpdateClasses, this will contain the class jarfile bytes
     * For AdHoc DDL work, this will be null
     * @param operationString The string for the catalog operation, if any. May be null in all cases.
     * For UpdateApplicationCatalog, this will contain the deployment string to apply
     * For UpdateClasses, this will contain the class deletion patterns
     * For AdHoc DDL work, this will be null
     */
public static CatalogChangeResult prepareApplicationCatalogDiff(String invocationName, final byte[] operationBytes, final String operationString, final String[] adhocDDLStmts, final byte[] replayHashOverride, final boolean isPromotion, final DrRoleType drRole, final boolean useAdhocDDL, boolean adminConnection, String hostname, String user) throws PrepareDiffFailureException {
    // create the change result and set up all the boiler plate
    CatalogChangeResult retval = new CatalogChangeResult();
    // ensure non-null
    retval.tablesThatMustBeEmpty = new String[0];
    retval.hasSchemaChange = true;
    if (replayHashOverride != null) {
        retval.isForReplay = true;
    }
    try {
        // catalog change specific boiler plate
        CatalogContext context = VoltDB.instance().getCatalogContext();
        // Start by assuming we're doing an @UpdateApplicationCatalog.  If-ladder below
        // will complete with newCatalogBytes actually containing the bytes of the
        // catalog to be applied, and deploymentString will contain an actual deployment string,
        // or null if it still needs to be filled in.
        InMemoryJarfile newCatalogJar = null;
        InMemoryJarfile oldJar = context.getCatalogJar().deepCopy();
        boolean updatedClass = false;
        String deploymentString = operationString;
        if ("@UpdateApplicationCatalog".equals(invocationName)) {
            // Grab the current catalog bytes if @UAC had a null catalog from deployment-only update
            if ((operationBytes == null) || (operationBytes.length == 0)) {
                newCatalogJar = oldJar;
            } else {
                newCatalogJar = CatalogUtil.loadInMemoryJarFile(operationBytes);
            }
        // If the deploymentString is null, we'll fill it in with current deployment later
        // Otherwise, deploymentString has the right contents, don't need to touch it
        } else if ("@UpdateClasses".equals(invocationName)) {
            // provided newCatalogJar is the jarfile with the new classes
            if (operationBytes != null) {
                newCatalogJar = new InMemoryJarfile(operationBytes);
            }
            try {
                InMemoryJarfile modifiedJar = modifyCatalogClasses(context.catalog, oldJar, operationString, newCatalogJar, drRole == DrRoleType.XDCR);
                if (modifiedJar == null) {
                    newCatalogJar = oldJar;
                } else {
                    newCatalogJar = modifiedJar;
                    updatedClass = true;
                }
            } catch (ClassNotFoundException e) {
                throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unexpected error in @UpdateClasses modifying classes from catalog: " + e.getMessage());
            }
            // Real deploymentString should be the current deployment, just set it to null
            // here and let it get filled in correctly later.
            deploymentString = null;
            // mark it as non-schema change
            retval.hasSchemaChange = false;
        } else if ("@AdHoc".equals(invocationName)) {
            // work.adhocDDLStmts should be applied to the current catalog
            try {
                newCatalogJar = addDDLToCatalog(context.catalog, oldJar, adhocDDLStmts, drRole == DrRoleType.XDCR);
            } catch (VoltCompilerException vce) {
                throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, vce.getMessage());
            } catch (IOException ioe) {
                throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, ioe.getMessage());
            } catch (Throwable t) {
                String msg = "Unexpected condition occurred applying DDL statements: " + t.toString();
                compilerLog.error(msg);
                throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, msg);
            }
            assert (newCatalogJar != null);
            if (newCatalogJar == null) {
                // Shouldn't ever get here
                String msg = "Unexpected failure in applying DDL statements to original catalog";
                compilerLog.error(msg);
                throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, msg);
            }
            // Real deploymentString should be the current deployment, just set it to null
            // here and let it get filled in correctly later.
            deploymentString = null;
        } else {
            // TODO: this if-chain doesn't feel like it even should exist
            assert (false);
        }
        // get the diff between catalogs
        // try to get the new catalog from the params
        Pair<InMemoryJarfile, String> loadResults = null;
        try {
            loadResults = CatalogUtil.loadAndUpgradeCatalogFromJar(newCatalogJar, drRole == DrRoleType.XDCR);
        } catch (IOException ioe) {
            // falling through to the ZOMG message in the big catch
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, ioe.getMessage());
        }
        retval.catalogBytes = loadResults.getFirst().getFullJarBytes();
        if (!retval.isForReplay) {
            retval.catalogHash = loadResults.getFirst().getSha1Hash();
        } else {
            retval.catalogHash = replayHashOverride;
        }
        String newCatalogCommands = CatalogUtil.getSerializedCatalogStringFromJar(loadResults.getFirst());
        retval.upgradedFromVersion = loadResults.getSecond();
        if (newCatalogCommands == null) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unable to read from catalog bytes");
        }
        Catalog newCatalog = new Catalog();
        newCatalog.execute(newCatalogCommands);
        // Retrieve the original deployment string, if necessary
        if (deploymentString == null) {
            // Go get the deployment string from the current catalog context
            byte[] deploymentBytes = context.getDeploymentBytes();
            if (deploymentBytes != null) {
                deploymentString = new String(deploymentBytes, Constants.UTF8ENCODING);
            }
            if (deploymentBytes == null || deploymentString == null) {
                throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "No deployment file provided and unable to recover previous deployment settings.");
            }
        }
        DeploymentType dt = CatalogUtil.parseDeploymentFromString(deploymentString);
        if (dt == null) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unable to update deployment configuration: Error parsing deployment string");
        }
        if (isPromotion && drRole == DrRoleType.REPLICA) {
            assert dt.getDr().getRole() == DrRoleType.REPLICA;
            dt.getDr().setRole(DrRoleType.MASTER);
        }
        String result = CatalogUtil.compileDeployment(newCatalog, dt, false);
        if (result != null) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "Unable to update deployment configuration: " + result);
        }
        //In non legacy mode discard the path element.
        if (!VoltDB.instance().isRunningWithOldVerbs()) {
            dt.setPaths(null);
        }
        //Always get deployment after its adjusted.
        retval.deploymentString = CatalogUtil.getDeployment(dt, true);
        retval.deploymentHash = CatalogUtil.makeDeploymentHash(retval.deploymentString.getBytes(Constants.UTF8ENCODING));
        // store the version of the catalog the diffs were created against.
        // verified when / if the update procedure runs in order to verify
        // catalogs only move forward
        retval.expectedCatalogVersion = context.catalogVersion;
        // compute the diff in StringBuilder
        CatalogDiffEngine diff = new CatalogDiffEngine(context.catalog, newCatalog);
        if (!diff.supported()) {
            throw new PrepareDiffFailureException(ClientResponse.GRACEFUL_FAILURE, "The requested catalog change(s) are not supported:\n" + diff.errors());
        }
        String commands = diff.commands();
        compilerLog.info(diff.getDescriptionOfChanges(updatedClass));
        retval.requireCatalogDiffCmdsApplyToEE = diff.requiresCatalogDiffCmdsApplyToEE();
        // since diff commands can be stupidly big, compress them here
        retval.encodedDiffCommands = Encoder.compressAndBase64Encode(commands);
        retval.diffCommandsLength = commands.length();
        String[][] emptyTablesAndReasons = diff.tablesThatMustBeEmpty();
        assert (emptyTablesAndReasons.length == 2);
        assert (emptyTablesAndReasons[0].length == emptyTablesAndReasons[1].length);
        retval.tablesThatMustBeEmpty = emptyTablesAndReasons[0];
        retval.reasonsForEmptyTables = emptyTablesAndReasons[1];
        retval.requiresSnapshotIsolation = diff.requiresSnapshotIsolation();
        retval.requiresNewExportGeneration = diff.requiresNewExportGeneration();
        retval.worksWithElastic = diff.worksWithElastic();
    } catch (PrepareDiffFailureException e) {
        throw e;
    } catch (Exception e) {
        String msg = "Unexpected error in adhoc or catalog update: " + e.getClass() + ", " + e.getMessage();
        compilerLog.warn(msg, e);
        throw new PrepareDiffFailureException(ClientResponse.UNEXPECTED_FAILURE, msg);
    }
    return retval;
}