Examples with JSONException org.json_voltpatches.JSONException used on opensource projects

Example 16 with JSONException

use of org.json_voltpatches.JSONException in project voltdb by VoltDB.

the class MaterializedViewProcessor method findBestMatchIndexForMatviewMinOrMax.

// if the materialized view has MIN / MAX, try to find an index defined on the source table
// covering all group by cols / exprs to avoid expensive tablescan.
// For now, the only acceptable index is defined exactly on the group by columns IN ORDER.
// This allows the same key to be used to do lookups on the grouped table index and the
// base table index.
// TODO: More flexible (but usually less optimal*) indexes may be allowed here and supported
// in the EE in the future including:
//   -- *indexes on the group keys listed out of order
//   -- *indexes on the group keys as a prefix before other indexed values.
//   -- (ENG-6511) indexes on the group keys PLUS the MIN/MAX argument value (to eliminate post-filtering)
// This function is mostly re-written for the fix of ENG-6511. --yzhang
private static Index findBestMatchIndexForMatviewMinOrMax(MaterializedViewInfo matviewinfo, Table srcTable, List<AbstractExpression> groupbyExprs, AbstractExpression minMaxAggExpr) {
    CatalogMap<Index> allIndexes = srcTable.getIndexes();
    StmtTableScan tableScan = new StmtTargetTableScan(srcTable);
    // Candidate index. If we can find an index covering both group-by columns and aggExpr (optimal) then we will
    // return immediately.
    // If the index found covers only group-by columns (sub-optimal), we will first cache it here.
    Index candidate = null;
    for (Index index : allIndexes) {
        // indexOptimalForMinMax == true if the index covered both the group-by columns and the min/max aggExpr.
        boolean indexOptimalForMinMax = false;
        // If minMaxAggExpr is not null, the diff can be zero or one.
        // Otherwise, for a usable index, its number of columns must agree with that of the group-by columns.
        final int diffAllowance = minMaxAggExpr == null ? 0 : 1;
        // Get all indexed exprs if there is any.
        String expressionjson = index.getExpressionsjson();
        List<AbstractExpression> indexedExprs = null;
        if (!expressionjson.isEmpty()) {
            try {
                indexedExprs = AbstractExpression.fromJSONArrayString(expressionjson, tableScan);
            } catch (JSONException e) {
                e.printStackTrace();
                assert (false);
                return null;
            }
        }
        // Get source table columns.
        List<Column> srcColumnArray = CatalogUtil.getSortedCatalogItems(srcTable.getColumns(), "index");
        MatViewIndexMatchingGroupby matchingCase = null;
        if (groupbyExprs == null) {
            // This means group-by columns are all simple columns.
            // It also means we can only access the group-by columns by colref.
            List<ColumnRef> groupbyColRefs = CatalogUtil.getSortedCatalogItems(matviewinfo.getGroupbycols(), "index");
            if (indexedExprs == null) {
                matchingCase = MatViewIndexMatchingGroupby.GB_COL_IDX_COL;
                // All the columns in the index are also simple columns, EASY! colref vs. colref
                List<ColumnRef> indexedColRefs = CatalogUtil.getSortedCatalogItems(index.getColumns(), "index");
                // indexedColRefs.size() == groupbyColRefs.size() + 1 (optimal, diffAllowance == 1)
                if (isInvalidIndexCandidate(indexedColRefs.size(), groupbyColRefs.size(), diffAllowance)) {
                    continue;
                }
                if (!isGroupbyMatchingIndex(matchingCase, groupbyColRefs, null, indexedColRefs, null, null)) {
                    continue;
                }
                if (isValidIndexCandidateForMinMax(indexedColRefs.size(), groupbyColRefs.size(), diffAllowance)) {
                    if (!isIndexOptimalForMinMax(matchingCase, minMaxAggExpr, indexedColRefs, null, srcColumnArray)) {
                        continue;
                    }
                    indexOptimalForMinMax = true;
                }
            } else {
                matchingCase = MatViewIndexMatchingGroupby.GB_COL_IDX_EXP;
                //                              for    index columns: convert    tve => col
                if (isInvalidIndexCandidate(indexedExprs.size(), groupbyColRefs.size(), diffAllowance)) {
                    continue;
                }
                if (!isGroupbyMatchingIndex(matchingCase, groupbyColRefs, null, null, indexedExprs, srcColumnArray)) {
                    continue;
                }
                if (isValidIndexCandidateForMinMax(indexedExprs.size(), groupbyColRefs.size(), diffAllowance)) {
                    if (!isIndexOptimalForMinMax(matchingCase, minMaxAggExpr, null, indexedExprs, null)) {
                        continue;
                    }
                    indexOptimalForMinMax = true;
                }
            }
        } else {
            matchingCase = MatViewIndexMatchingGroupby.GB_EXP_IDX_EXP;
            // AND, indexedExprs must not be null in this case. (yeah!)
            if (indexedExprs == null) {
                continue;
            }
            if (isInvalidIndexCandidate(indexedExprs.size(), groupbyExprs.size(), diffAllowance)) {
                continue;
            }
            if (!isGroupbyMatchingIndex(matchingCase, null, groupbyExprs, null, indexedExprs, null)) {
                continue;
            }
            if (isValidIndexCandidateForMinMax(indexedExprs.size(), groupbyExprs.size(), diffAllowance)) {
                if (!isIndexOptimalForMinMax(matchingCase, minMaxAggExpr, null, indexedExprs, null)) {
                    continue;
                }
                indexOptimalForMinMax = true;
            }
        }
        // NOW index at least covered all group-by columns (sub-optimal candidate)
        if (!index.getPredicatejson().isEmpty()) {
            // Additional check for partial indexes to make sure matview WHERE clause
            // covers the partial index predicate
            List<AbstractExpression> coveringExprs = new ArrayList<>();
            List<AbstractExpression> exactMatchCoveringExprs = new ArrayList<>();
            try {
                String encodedPredicate = matviewinfo.getPredicate();
                if (!encodedPredicate.isEmpty()) {
                    String predicate = Encoder.hexDecodeToString(encodedPredicate);
                    AbstractExpression matViewPredicate = AbstractExpression.fromJSONString(predicate, tableScan);
                    coveringExprs.addAll(ExpressionUtil.uncombineAny(matViewPredicate));
                }
            } catch (JSONException e) {
                e.printStackTrace();
                assert (false);
                return null;
            }
            String predicatejson = index.getPredicatejson();
            if (!predicatejson.isEmpty() && !SubPlanAssembler.isPartialIndexPredicateCovered(tableScan, coveringExprs, predicatejson, exactMatchCoveringExprs)) {
                // where clause -- give up on this index
                continue;
            }
        }
        // it is already the best index we can get, return immediately.
        if (indexOptimalForMinMax) {
            return index;
        }
        // otherwise wait to see if we can find something better!
        candidate = index;
    }
    return candidate;
}

Example 17 with JSONException

use of org.json_voltpatches.JSONException in project voltdb by VoltDB.

the class MaterializedViewProcessor method setGroupedTablePartitionColumn.

private void setGroupedTablePartitionColumn(MaterializedViewInfo mvi, Column partitionColumn) throws VoltCompilerException {
    // A view of a replicated table is replicated.
    // A view of a partitioned table is partitioned -- regardless of whether it has a partition key
    // -- it certainly isn't replicated!
    // If the partitioning column is grouped, its counterpart is the partitioning column of the view table.
    // Otherwise, the view table just doesn't have a partitioning column
    // -- it is seemingly randomly distributed,
    // and its grouped columns are only locally unique but not globally unique.
    Table destTable = mvi.getDest();
    // Get the grouped columns in "index" order.
    // This order corresponds to the iteration order of the MaterializedViewInfo's group by columns.
    List<Column> destColumnArray = CatalogUtil.getSortedCatalogItems(destTable.getColumns(), "index");
    // Note getTypeName gets the column name -- go figure.
    String partitionColName = partitionColumn.getTypeName();
    if (mvi.getGroupbycols().size() > 0) {
        int index = 0;
        for (ColumnRef cref : CatalogUtil.getSortedCatalogItems(mvi.getGroupbycols(), "index")) {
            Column srcCol = cref.getColumn();
            if (srcCol.getName().equals(partitionColName)) {
                Column destCol = destColumnArray.get(index);
                destTable.setPartitioncolumn(destCol);
                return;
            }
            ++index;
        }
    } else {
        String complexGroupbyJson = mvi.getGroupbyexpressionsjson();
        if (complexGroupbyJson.length() > 0) {
            int partitionColIndex = partitionColumn.getIndex();
            List<AbstractExpression> mvComplexGroupbyCols = null;
            try {
                mvComplexGroupbyCols = AbstractExpression.fromJSONArrayString(complexGroupbyJson, null);
            } catch (JSONException e) {
                e.printStackTrace();
            }
            int index = 0;
            for (AbstractExpression expr : mvComplexGroupbyCols) {
                if (expr instanceof TupleValueExpression) {
                    TupleValueExpression tve = (TupleValueExpression) expr;
                    if (tve.getColumnIndex() == partitionColIndex) {
                        Column destCol = destColumnArray.get(index);
                        destTable.setPartitioncolumn(destCol);
                        return;
                    }
                }
                ++index;
            }
        }
    }
}

Example 18 with JSONException

use of org.json_voltpatches.JSONException in project voltdb by VoltDB.

the class Distributer method updateProcedurePartitioning.

private void updateProcedurePartitioning(VoltTable vt) {
    Map<String, Procedure> procs = Maps.newHashMap();
    while (vt.advanceRow()) {
        try {
            //Data embedded in JSON object in remarks column
            String jsString = vt.getString(6);
            String procedureName = vt.getString(2);
            JSONObject jsObj = new JSONObject(jsString);
            boolean readOnly = jsObj.getBoolean(Constants.JSON_READ_ONLY);
            if (jsObj.getBoolean(Constants.JSON_SINGLE_PARTITION)) {
                int partitionParameter = jsObj.getInt(Constants.JSON_PARTITION_PARAMETER);
                int partitionParameterType = jsObj.getInt(Constants.JSON_PARTITION_PARAMETER_TYPE);
                procs.put(procedureName, new Procedure(false, readOnly, partitionParameter, partitionParameterType));
            } else {
                // Multi Part procedure JSON descriptors omit the partitionParameter
                procs.put(procedureName, new Procedure(true, readOnly, Procedure.PARAMETER_NONE, Procedure.PARAMETER_NONE));
            }
        } catch (JSONException e) {
            e.printStackTrace();
        }
    }
    ImmutableSortedMap<String, Procedure> oldProcs = m_procedureInfo.get();
    m_procedureInfo.compareAndSet(oldProcs, ImmutableSortedMap.copyOf(procs));
}

Example 19 with JSONException

use of org.json_voltpatches.JSONException in project voltdb by VoltDB.

the class LeaderAppointer method assignLeader.

private long assignLeader(int partitionId, List<Long> children) {
    // We used masterHostId = -1 as a way to force the leader choice to be
    // the first replica in the list, if we don't have some other mechanism
    // which has successfully overridden it.
    int masterHostId = -1;
    if (m_state.get() == AppointerState.CLUSTER_START) {
        try {
            // find master in topo
            JSONArray parts = m_topo.getJSONArray(AbstractTopology.TOPO_PARTITIONS);
            for (int p = 0; p < parts.length(); p++) {
                JSONObject aPartition = parts.getJSONObject(p);
                int pid = aPartition.getInt(AbstractTopology.TOPO_PARTITION_ID);
                if (pid == partitionId) {
                    masterHostId = aPartition.getInt(AbstractTopology.TOPO_MASTER);
                    break;
                }
            }
        } catch (JSONException jse) {
            tmLog.error("Failed to find master for partition " + partitionId + ", defaulting to 0");
            jse.printStackTrace();
            // stupid default
            masterHostId = -1;
        }
    } else {
        // For now, if we're appointing a new leader as a result of a
        // failure, just pick the first replica in the children list.
        // Could eventually do something more complex here to try to keep a
        // semi-balance, but it's unclear that this has much utility until
        // we add rebalancing on rejoin as well.
        masterHostId = -1;
    }
    long masterHSId = children.get(0);
    for (Long child : children) {
        if (CoreUtils.getHostIdFromHSId(child) == masterHostId) {
            masterHSId = child;
            break;
        }
    }
    tmLog.info("Appointing HSId " + CoreUtils.hsIdToString(masterHSId) + " as leader for partition " + partitionId);
    try {
        m_iv2appointees.put(partitionId, masterHSId);
    } catch (Exception e) {
        VoltDB.crashLocalVoltDB("Unable to appoint new master for partition " + partitionId, true, e);
    }
    return masterHSId;
}

Example 20 with JSONException

use of org.json_voltpatches.JSONException in project voltdb by VoltDB.

the class MaterializedViewFixInfo method processMVBasedQueryFix.

/**
     * Check whether the results from a materialized view need to be
     * re-aggregated on the coordinator by the view's GROUP BY columns
     * prior to any of the processing specified by the query.
     * This is normally the case when a mat view's source table is partitioned
     * and the view's GROUP BY does not include the partition key.
     * There is a special edge case where the query already contains the exact
     * reaggregations that the added-cost fix would introduce, so the fix can
     * be skipped as an optimization.
     * Set the m_needed flag to true, only if the reaggregation fix is needed.
     * @return The value of m_needed
     */
public boolean processMVBasedQueryFix(StmtTableScan mvTableScan, Set<SchemaColumn> scanColumns, JoinNode joinTree, List<ParsedColInfo> displayColumns, List<ParsedColInfo> groupByColumns) {
    //@TODO
    if (!(mvTableScan instanceof StmtTargetTableScan)) {
        return false;
    }
    Table table = ((StmtTargetTableScan) mvTableScan).getTargetTable();
    assert (table != null);
    String mvTableName = table.getTypeName();
    Table srcTable = table.getMaterializer();
    if (srcTable == null) {
        return false;
    }
    if (table.getIsreplicated()) {
        return false;
    }
    // Justify whether partition column is in group by column list or not
    if (table.getPartitioncolumn() != null) {
        return false;
    }
    m_mvTableScan = mvTableScan;
    Set<String> mvDDLGroupbyColumnNames = new HashSet<>();
    List<Column> mvColumnArray = CatalogUtil.getSortedCatalogItems(table.getColumns(), "index");
    String mvTableAlias = getMVTableAlias();
    // Get the number of group-by columns.
    int numOfGroupByColumns;
    MaterializedViewInfo mvInfo = srcTable.getViews().get(mvTableName);
    if (mvInfo != null) {
        // single table view
        String complexGroupbyJson = mvInfo.getGroupbyexpressionsjson();
        if (complexGroupbyJson.length() > 0) {
            List<AbstractExpression> mvComplexGroupbyCols = null;
            try {
                mvComplexGroupbyCols = AbstractExpression.fromJSONArrayString(complexGroupbyJson, null);
            } catch (JSONException e) {
                e.printStackTrace();
            }
            numOfGroupByColumns = mvComplexGroupbyCols.size();
        } else {
            numOfGroupByColumns = mvInfo.getGroupbycols().size();
        }
    } else {
        // joined table view
        MaterializedViewHandlerInfo mvHandlerInfo = table.getMvhandlerinfo().get("mvHandlerInfo");
        numOfGroupByColumns = mvHandlerInfo.getGroupbycolumncount();
    }
    if (scanColumns.isEmpty() && numOfGroupByColumns == 0) {
        // This is an edge case that can happen if the view
        // has no group by keys, and we are just
        // doing a count(*) on the output of the view.
        //
        // Having no GB keys or scan columns would cause us to
        // produce plan nodes that have a 0-column output schema.
        // We can't handle this in several places, so add the
        // count(*) column from the view to the scan columns.
        // this is the "count(*)" column.
        Column mvCol = mvColumnArray.get(0);
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, 0);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        String colName = mvCol.getName();
        SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
        scanColumns.add(scol);
    }
    // Start to do real materialized view processing to fix the duplicates problem.
    // (1) construct new projection columns for scan plan node.
    Set<SchemaColumn> mvDDLGroupbyColumns = new HashSet<>();
    NodeSchema inlineProjSchema = new NodeSchema();
    for (SchemaColumn scol : scanColumns) {
        inlineProjSchema.addColumn(scol);
    }
    for (int i = 0; i < numOfGroupByColumns; i++) {
        Column mvCol = mvColumnArray.get(i);
        String colName = mvCol.getName();
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, i);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        mvDDLGroupbyColumnNames.add(colName);
        SchemaColumn scol = new SchemaColumn(mvTableName, mvTableAlias, colName, colName, tve);
        mvDDLGroupbyColumns.add(scol);
        if (!scanColumns.contains(scol)) {
            scanColumns.add(scol);
            // construct new projection columns for scan plan node.
            inlineProjSchema.addColumn(scol);
        }
    }
    // Record the re-aggregation type for each scan columns.
    Map<String, ExpressionType> mvColumnReAggType = new HashMap<>();
    for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
        Column mvCol = mvColumnArray.get(i);
        ExpressionType reAggType = ExpressionType.get(mvCol.getAggregatetype());
        if (reAggType == ExpressionType.AGGREGATE_COUNT_STAR || reAggType == ExpressionType.AGGREGATE_COUNT) {
            reAggType = ExpressionType.AGGREGATE_SUM;
        }
        mvColumnReAggType.put(mvCol.getName(), reAggType);
    }
    assert (inlineProjSchema.size() > 0);
    m_scanInlinedProjectionNode = new ProjectionPlanNode(inlineProjSchema);
    // (2) Construct the reAggregation Node.
    // Construct the reAggregation plan node's aggSchema
    m_reAggNode = new HashAggregatePlanNode();
    int outputColumnIndex = 0;
    // inlineProjSchema contains the group by columns, while aggSchema may do not.
    NodeSchema aggSchema = new NodeSchema();
    // Construct reAggregation node's aggregation and group by list.
    for (SchemaColumn scol : inlineProjSchema.getColumns()) {
        if (mvDDLGroupbyColumns.contains(scol)) {
            // Add group by expression.
            m_reAggNode.addGroupByExpression(scol.getExpression());
        } else {
            ExpressionType reAggType = mvColumnReAggType.get(scol.getColumnName());
            assert (reAggType != null);
            AbstractExpression agg_input_expr = scol.getExpression();
            assert (agg_input_expr instanceof TupleValueExpression);
            // Add aggregation information.
            m_reAggNode.addAggregate(reAggType, false, outputColumnIndex, agg_input_expr);
        }
        aggSchema.addColumn(scol);
        outputColumnIndex++;
    }
    assert (aggSchema.size() > 0);
    m_reAggNode.setOutputSchema(aggSchema);
    // Collect all TVEs that need to be do re-aggregation in coordinator.
    List<TupleValueExpression> needReAggTVEs = new ArrayList<>();
    List<AbstractExpression> aggPostExprs = new ArrayList<>();
    for (int i = numOfGroupByColumns; i < mvColumnArray.size(); i++) {
        Column mvCol = mvColumnArray.get(i);
        TupleValueExpression tve = new TupleValueExpression(mvTableName, mvTableAlias, mvCol, -1);
        tve.setOrigStmtId(mvTableScan.getStatementId());
        needReAggTVEs.add(tve);
    }
    collectReAggNodePostExpressions(joinTree, needReAggTVEs, aggPostExprs);
    AbstractExpression aggPostExpr = ExpressionUtil.combinePredicates(aggPostExprs);
    // Add post filters for the reAggregation node.
    m_reAggNode.setPostPredicate(aggPostExpr);
    // ENG-5386
    if (m_edgeCaseQueryNoFixNeeded && edgeCaseQueryNoFixNeeded(mvDDLGroupbyColumnNames, mvColumnReAggType, displayColumns, groupByColumns)) {
        return false;
    }
    m_needed = true;
    return true;
}