use of org.json_voltpatches.JSONException in project voltdb by VoltDB.
the class InvocationDispatcher method useSnapshotCatalogToRestoreSnapshotSchema.
private final ClientResponseImpl useSnapshotCatalogToRestoreSnapshotSchema(final StoredProcedureInvocation task, final InvocationClientHandler handler, final Connection ccxn, final AuthUser user, OverrideCheck bypass) {
CatalogContext catalogContext = m_catalogContext.get();
if (!catalogContext.cluster.getUseddlschema()) {
return gracefulFailureResponse("Cannot restore catalog from snapshot when schema is set to catalog in the deployment.", task.clientHandle);
}
log.info("No schema found. Restoring schema and procedures from snapshot.");
try {
JSONObject jsObj = new JSONObject(task.getParams().getParam(0).toString());
final File catalogFH = getSnapshotCatalogFile(jsObj);
final byte[] catalog;
try {
catalog = MiscUtils.fileToBytes(catalogFH);
} catch (IOException e) {
log.warn("Unable to access catalog file " + catalogFH, e);
return unexpectedFailureResponse("Unable to access catalog file " + catalogFH, task.clientHandle);
}
final String dep = new String(catalogContext.getDeploymentBytes(), StandardCharsets.UTF_8);
final StoredProcedureInvocation catalogUpdateTask = new StoredProcedureInvocation();
catalogUpdateTask.setProcName("@UpdateApplicationCatalog");
catalogUpdateTask.setParams(catalog, dep);
//A connection with positive id will be thrown into live client statistics. The connection does not support stats.
//Thus make the connection id as a negative constant to skip the stats collection.
final SimpleClientResponseAdapter alternateAdapter = new SimpleClientResponseAdapter(ClientInterface.RESTORE_SCHEMAS_CID, "Empty database snapshot restore catalog update");
final InvocationClientHandler alternateHandler = new InvocationClientHandler() {
@Override
public boolean isAdmin() {
return handler.isAdmin();
}
@Override
public long connectionId() {
return ClientInterface.RESTORE_SCHEMAS_CID;
}
};
final long sourceHandle = task.clientHandle;
SimpleClientResponseAdapter.SyncCallback restoreCallback = new SimpleClientResponseAdapter.SyncCallback();
final ListenableFuture<ClientResponse> onRestoreComplete = restoreCallback.getResponseFuture();
onRestoreComplete.addListener(new Runnable() {
@Override
public void run() {
ClientResponse r;
try {
r = onRestoreComplete.get();
} catch (ExecutionException | InterruptedException e) {
VoltDB.crashLocalVoltDB("Should never happen", true, e);
return;
}
transmitResponseMessage(r, ccxn, sourceHandle);
}
}, CoreUtils.SAMETHREADEXECUTOR);
task.setClientHandle(alternateAdapter.registerCallback(restoreCallback));
SimpleClientResponseAdapter.SyncCallback catalogUpdateCallback = new SimpleClientResponseAdapter.SyncCallback();
final ListenableFuture<ClientResponse> onCatalogUpdateComplete = catalogUpdateCallback.getResponseFuture();
onCatalogUpdateComplete.addListener(new Runnable() {
@Override
public void run() {
ClientResponse r;
try {
r = onCatalogUpdateComplete.get();
} catch (ExecutionException | InterruptedException e) {
VoltDB.crashLocalVoltDB("Should never happen", true, e);
return;
}
if (r.getStatus() != ClientResponse.SUCCESS) {
transmitResponseMessage(r, ccxn, sourceHandle);
log.error("Received error response for updating catalog " + r.getStatusString());
return;
}
m_catalogContext.set(VoltDB.instance().getCatalogContext());
dispatch(task, alternateHandler, alternateAdapter, user, bypass, false);
}
}, CoreUtils.SAMETHREADEXECUTOR);
catalogUpdateTask.setClientHandle(alternateAdapter.registerCallback(catalogUpdateCallback));
VoltDB.instance().getClientInterface().bindAdapter(alternateAdapter, null);
// dispatch the catalog update
dispatchNTProcedure(alternateHandler, catalogUpdateTask, user, alternateAdapter, System.nanoTime(), false);
} catch (JSONException e) {
return unexpectedFailureResponse("Unable to parse parameters.", task.clientHandle);
}
return null;
}
use of org.json_voltpatches.JSONException in project voltdb by VoltDB.
the class InvocationDispatcher method takeShutdownSaveSnapshot.
private final ClientResponseImpl takeShutdownSaveSnapshot(final StoredProcedureInvocation task, final InvocationClientHandler handler, final Connection ccxn, final AuthUser user, OverrideCheck bypass) {
// shutdown save snapshot is available for Pro edition only
if (!MiscUtils.isPro()) {
task.setParams();
return dispatch(task, handler, ccxn, user, bypass, false);
}
Object p0 = task.getParams().getParam(0);
final long zkTxnId;
if (p0 instanceof Long) {
zkTxnId = ((Long) p0).longValue();
} else if (p0 instanceof String) {
try {
zkTxnId = Long.parseLong((String) p0);
} catch (NumberFormatException e) {
return gracefulFailureResponse("Incorrect argument type", task.clientHandle);
}
} else {
return gracefulFailureResponse("Incorrect argument type", task.clientHandle);
}
VoltDBInterface voltdb = VoltDB.instance();
if (!voltdb.isPreparingShuttingdown()) {
log.warn("Ignoring shutdown save snapshot request as VoltDB is not shutting down");
return unexpectedFailureResponse("Ignoring shutdown save snapshot request as VoltDB is not shutting down", task.clientHandle);
}
final ZooKeeper zk = voltdb.getHostMessenger().getZK();
// network threads are blocked from making zookeeper calls
Future<Long> fut = voltdb.getSES(true).submit(new Callable<Long>() {
@Override
public Long call() {
try {
Stat stat = zk.exists(VoltZK.operationMode, false);
if (stat == null) {
VoltDB.crashLocalVoltDB("cluster operation mode zookeeper node does not exist");
return Long.MIN_VALUE;
}
return stat.getMzxid();
} catch (KeeperException | InterruptedException e) {
VoltDB.crashLocalVoltDB("Failed to stat the cluster operation zookeeper node", true, e);
return Long.MIN_VALUE;
}
}
});
try {
if (fut.get().longValue() != zkTxnId) {
return unexpectedFailureResponse("Internal error: cannot write a startup snapshot because the " + "current system state is not consistent with an orderly shutdown. " + "Please try \"voltadmin shutdown --save\" again.", task.clientHandle);
}
} catch (InterruptedException | ExecutionException e1) {
VoltDB.crashLocalVoltDB("Failed to stat the cluster operation zookeeper node", true, e1);
return null;
}
NodeSettings paths = m_catalogContext.get().getNodeSettings();
String data;
try {
data = new JSONStringer().object().keySymbolValuePair(SnapshotUtil.JSON_TERMINUS, zkTxnId).endObject().toString();
} catch (JSONException e) {
VoltDB.crashLocalVoltDB("Failed to create startup snapshot save command", true, e);
return null;
}
log.info("Saving startup snapshot");
consoleLog.info("Taking snapshot to save database contents");
final SimpleClientResponseAdapter alternateAdapter = new SimpleClientResponseAdapter(ClientInterface.SHUTDONW_SAVE_CID, "Blocking Startup Snapshot Save");
final InvocationClientHandler alternateHandler = new InvocationClientHandler() {
@Override
public boolean isAdmin() {
return handler.isAdmin();
}
@Override
public long connectionId() {
return ClientInterface.SHUTDONW_SAVE_CID;
}
};
final long sourceHandle = task.clientHandle;
task.setClientHandle(alternateAdapter.registerCallback(SimpleClientResponseAdapter.NULL_CALLBACK));
SnapshotUtil.SnapshotResponseHandler savCallback = new SnapshotUtil.SnapshotResponseHandler() {
@Override
public void handleResponse(ClientResponse r) {
if (r == null) {
String msg = "Snapshot save failed. The database is paused and the shutdown has been cancelled";
transmitResponseMessage(gracefulFailureResponse(msg, sourceHandle), ccxn, sourceHandle);
}
if (r.getStatus() != ClientResponse.SUCCESS) {
String msg = "Snapshot save failed: " + r.getStatusString() + ". The database is paused and the shutdown has been cancelled";
ClientResponseImpl resp = new ClientResponseImpl(ClientResponse.GRACEFUL_FAILURE, r.getResults(), msg, sourceHandle);
transmitResponseMessage(resp, ccxn, sourceHandle);
}
consoleLog.info("Snapshot taken successfully");
task.setParams();
dispatch(task, alternateHandler, alternateAdapter, user, bypass, false);
}
};
// network threads are blocked from making zookeeper calls
final byte[] guardContent = data.getBytes(StandardCharsets.UTF_8);
Future<Boolean> guardFuture = voltdb.getSES(true).submit(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
try {
ZKUtil.asyncMkdirs(zk, VoltZK.shutdown_save_guard, guardContent).get();
} catch (NodeExistsException itIsOk) {
return false;
} catch (InterruptedException | KeeperException e) {
VoltDB.crashLocalVoltDB("Failed to create shutdown save guard zookeeper node", true, e);
return false;
}
return true;
}
});
boolean created;
try {
created = guardFuture.get().booleanValue();
} catch (InterruptedException | ExecutionException e) {
VoltDB.crashLocalVoltDB("Failed to create shutdown save guard zookeeper node", true, e);
return null;
}
if (!created) {
return unexpectedFailureResponse("Internal error: detected concurrent invocations of \"voltadmin shutdown --save\"", task.clientHandle);
}
voltdb.getClientInterface().bindAdapter(alternateAdapter, null);
SnapshotUtil.requestSnapshot(sourceHandle, paths.resolve(paths.getSnapshoth()).toPath().toUri().toString(), SnapshotUtil.getShutdownSaveNonce(zkTxnId), true, SnapshotFormat.NATIVE, SnapshotPathType.SNAP_AUTO, data, savCallback, true);
return null;
}
use of org.json_voltpatches.JSONException in project voltdb by VoltDB.
the class ExtensibleSnapshotDigestData method writeExportSequenceNumbersToSnapshot.
private void writeExportSequenceNumbersToSnapshot(JSONStringer stringer) throws IOException {
try {
stringer.key("exportSequenceNumbers").array();
for (Map.Entry<String, Map<Integer, Pair<Long, Long>>> entry : m_exportSequenceNumbers.entrySet()) {
stringer.object();
stringer.keySymbolValuePair("exportTableName", entry.getKey());
stringer.key("sequenceNumberPerPartition").array();
for (Map.Entry<Integer, Pair<Long, Long>> sequenceNumber : entry.getValue().entrySet()) {
stringer.object();
stringer.keySymbolValuePair("partition", sequenceNumber.getKey());
//First value is the ack offset which matters for pauseless rejoin, but not persistence
stringer.keySymbolValuePair("exportSequenceNumber", sequenceNumber.getValue().getSecond());
stringer.endObject();
}
stringer.endArray();
stringer.endObject();
}
stringer.endArray();
} catch (JSONException e) {
throw new IOException(e);
}
}
use of org.json_voltpatches.JSONException in project voltdb by VoltDB.
the class MpScheduler method getBalancePartitions.
/**
* Extract the two involved partitions from the @BalancePartitions request.
*/
private Set<Integer> getBalancePartitions(Iv2InitiateTaskMessage msg) {
try {
JSONObject jsObj = new JSONObject((String) msg.getParameters()[0]);
BalancePartitionsRequest request = new BalancePartitionsRequest(jsObj);
return Sets.newHashSet(request.partitionPairs.get(0).srcPartition, request.partitionPairs.get(0).destPartition);
} catch (JSONException e) {
hostLog.warn("Unable to determine partitions for @BalancePartitions", e);
return null;
}
}
use of org.json_voltpatches.JSONException in project voltdb by VoltDB.
the class ReplaceWithIndexLimit method recursivelyApply.
@Override
protected AbstractPlanNode recursivelyApply(AbstractPlanNode plan) {
assert (plan != null);
// depth first:
// Find AggregatePlanNode with exactly one child
// where that child is an AbstractScanPlanNode.
// Replace qualifying SeqScanPlanNode with an
// IndexScanPlanNode with an inlined LimitPlanNode;
// or appending the LimitPlanNode to the existing
// qualified IndexScanPlanNode.
ArrayList<AbstractPlanNode> children = new ArrayList<AbstractPlanNode>();
for (int i = 0; i < plan.getChildCount(); i++) children.add(plan.getChild(i));
for (AbstractPlanNode child : children) {
// TODO this will break when children feed multiple parents
AbstractPlanNode newChild = recursivelyApply(child);
// Do a graft into the (parent) plan only if a replacement for a child was found.
if (newChild == child) {
continue;
}
child.removeFromGraph();
plan.addAndLinkChild(newChild);
}
// check for an aggregation of the right form
if ((plan instanceof AggregatePlanNode) == false)
return plan;
assert (plan.getChildCount() == 1);
AggregatePlanNode aggplan = (AggregatePlanNode) plan;
// handle one single min() / max() now
// TODO: combination of [min(), max(), count()]
SortDirectionType sortDirection = SortDirectionType.INVALID;
if (aggplan.isTableMin()) {
sortDirection = SortDirectionType.ASC;
} else if (aggplan.isTableMax()) {
sortDirection = SortDirectionType.DESC;
} else {
return plan;
}
AbstractPlanNode child = plan.getChild(0);
AbstractExpression aggExpr = aggplan.getFirstAggregateExpression();
// for a SEQSCAN, replace it with a INDEXSCAN node with an inline LIMIT plan node
if (child instanceof SeqScanPlanNode) {
// should have other index access plan if any qualified index found for the predicate
if (((SeqScanPlanNode) child).getPredicate() != null) {
return plan;
}
if (((AbstractScanPlanNode) child).isSubQuery()) {
return plan;
}
// create an empty bindingExprs list, used for store (possible) bindings for adHoc query
ArrayList<AbstractExpression> bindings = new ArrayList<AbstractExpression>();
Index ret = findQualifiedIndex(((SeqScanPlanNode) child), aggExpr, bindings);
if (ret == null) {
return plan;
} else {
// 1. create one INDEXSCAN plan node with inlined LIMIT
// and replace the SEQSCAN node with it
// 2. we know which end row we want to fetch, so it's safe to
// specify sorting direction here
IndexScanPlanNode ispn = new IndexScanPlanNode((SeqScanPlanNode) child, aggplan, ret, sortDirection);
ispn.setBindings(bindings);
assert (ispn.getSearchKeyExpressions().size() == 0);
if (sortDirection == SortDirectionType.ASC) {
assert (aggplan.isTableMin());
ispn.setSkipNullPredicate(0);
}
LimitPlanNode lpn = new LimitPlanNode();
lpn.setLimit(1);
lpn.setOffset(0);
ispn.addInlinePlanNode(lpn);
// remove old SeqScan node and link the new generated IndexScan node
plan.clearChildren();
plan.addAndLinkChild(ispn);
return plan;
}
}
if ((child instanceof IndexScanPlanNode) == false) {
return plan;
}
// already have the IndexScanPlanNode
IndexScanPlanNode ispn = (IndexScanPlanNode) child;
// we added for reverse scan purpose only
if (((IndexScanPlanNode) child).getPredicate() != null && !((IndexScanPlanNode) child).isPredicatesOptimizableForAggregate()) {
return plan;
}
// Guard against (possible future?) cases of indexable subquery.
if (((AbstractScanPlanNode) child).isSubQuery()) {
return plan;
}
// 2. Handle equality filters and one other comparison operator (<, <=, >, >=), see comments below
if (ispn.getLookupType() != IndexLookupType.EQ && Math.abs(ispn.getSearchKeyExpressions().size() - ExpressionUtil.uncombinePredicate(ispn.getEndExpression()).size()) > 1) {
return plan;
}
// exprs will be used as filterExprs to check the index
// For forward scan, the initial value is endExprs and might be changed in different values in variant cases
// For reverse scan, the initial value is initialExprs which is the "old" endExprs
List<AbstractExpression> exprs;
int numOfSearchKeys = ispn.getSearchKeyExpressions().size();
if (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE) {
exprs = ExpressionUtil.uncombinePredicate(ispn.getInitialExpression());
numOfSearchKeys -= 1;
} else {
exprs = ExpressionUtil.uncombinePredicate(ispn.getEndExpression());
}
int numberOfExprs = exprs.size();
/* Retrieve the index expressions from the target index. (ENG-8819, Ethan)
* This is because we found that for the following two queries:
* #1: explain select max(c2/2) from t where c1=1 and c2/2<=3;
* #2: explain select max(c2/2) from t where c1=1 and c2/2<=?;
* We can get an inline limit 1 for #2 but not for #1. This is because all constants in #1 got parameterized.
* The result is that the query cannot pass the bindingToIndexedExpression() tests below
* because we lost all the constant value expressions (cannot attempt to bind a pve to a pve!).
* Those constant values expressions can only be accessed from the idnex.
* We will not add those bindings to the ispn.getBindings() here because they will be added anyway in checkIndex().
* PS: For this case (i.e. index on expressions), checkIndex() will call checkExpressionIndex(),
* where bindings will be added.
*/
Index indexToUse = ispn.getCatalogIndex();
String tableAlias = ispn.getTargetTableAlias();
List<AbstractExpression> indexedExprs = null;
if (!indexToUse.getExpressionsjson().isEmpty()) {
StmtTableScan tableScan = m_parsedStmt.getStmtTableScanByAlias(tableAlias);
try {
indexedExprs = AbstractExpression.fromJSONArrayString(indexToUse.getExpressionsjson(), tableScan);
} catch (JSONException e) {
e.printStackTrace();
assert (false);
return plan;
}
}
/* If there is only 1 difference between searchkeyExprs and endExprs,
* 1. trivial filters can be discarded, 2 possibilities:
* a. SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ?
* <=> SELECT MIN(X) FROM T WHERE [other prefix filters] && the X < / <= ? filter
* b. SELECT MAX(X) FROM T WHERE X > / >= ?
* <=> SELECT MAX(X) FROM T with post-filter
* 2. filter should act as equality filter, 2 possibilities
* SELECT MIN(X) FROM T WHERE [other prefix filters] X > / >= ?
* SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?
* check if there is other filters for SELECT MAX(X) FROM T WHERE [other prefix filter AND ] X > / >= ?
* but we should allow SELECT MAX(X) FROM T WHERE X = ?
* This is for queries having MAX() but no ORDER BY. (ENG-8819, Ethan)
* sortDirection == DESC if max, ASC if min. ispn.getSortDirection() == INVALID if no ORDER BY. */
if (sortDirection == SortDirectionType.DESC && ispn.getSortDirection() == SortDirectionType.INVALID) {
/* numberOfExprs = exprs.size(), exprs are initial expressions for reversed index scans (lookupType LT, LTE),
* are end expressions for forward index scans (lookupType GT, GTE, EQ).
* Note, lookupType doesn't decide the scan direction for sure. MIN(X) where X < ? is still a forward scan.
* X < ? will be a post filter for the scan rather than an initial expression. */
if (numberOfExprs == 1) {
// e.g.: explain select max(c2/2) from t where c2/2<=3;
// In this case, as long as the where condition (exprs.get(0)) matches the aggregation argument, continue.
AbstractExpression exprToBind = indexedExprs == null ? exprs.get(0).getLeft() : indexedExprs.get(0);
if (aggExpr.bindingToIndexedExpression(exprToBind) == null) {
return plan;
}
} else if (numberOfExprs > 1) {
// ENG-4016: Optimization for query SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?
// Just keep trying, don't return early.
boolean earlyReturn = true;
for (int i = 0; i < numberOfExprs; ++i) {
AbstractExpression expr = exprs.get(i);
AbstractExpression indexedExpr = indexedExprs == null ? expr.getLeft() : indexedExprs.get(i);
if (aggExpr.bindingToIndexedExpression(indexedExpr) != null && (expr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO || expr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || expr.getExpressionType() == ExpressionType.COMPARE_EQUAL)) {
earlyReturn = false;
break;
}
}
if (earlyReturn) {
return plan;
}
}
}
// have an upper bound: # of endingExpr is more than # of searchExpr
if (numberOfExprs > numOfSearchKeys) {
AbstractExpression lastEndExpr = exprs.get(numberOfExprs - 1);
// check last ending condition, see whether it is
// SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ? or
// other filters will be checked later
AbstractExpression exprToBind = indexedExprs == null ? lastEndExpr.getLeft() : indexedExprs.get(numberOfExprs - 1);
if ((lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO) && aggExpr.bindingToIndexedExpression(exprToBind) != null) {
exprs.remove(lastEndExpr);
}
}
// and we can take advantage of that
if (checkIndex(ispn.getCatalogIndex(), aggExpr, exprs, ispn.getBindings(), tableAlias)) {
// we know which end we want to fetch, set the sort direction
ispn.setSortDirection(sortDirection);
// for SELECT MIN(X) FROM T WHERE [prefix filters] = ?
if (numberOfExprs == numOfSearchKeys && sortDirection == SortDirectionType.ASC) {
if (ispn.getLookupType() == IndexLookupType.GTE) {
assert (aggplan.isTableMin());
ispn.setSkipNullPredicate(numOfSearchKeys);
}
}
// reset the IndexLookupType, remove "added" searchKey, add back to endExpression, and clear "added" predicate
if (sortDirection == SortDirectionType.ASC && (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE)) {
ispn.setLookupType(IndexLookupType.GTE);
ispn.removeLastSearchKey();
ispn.addEndExpression(ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).get(numberOfExprs - 1));
ispn.setSkipNullPredicate(numOfSearchKeys);
ispn.resetPredicate();
}
// add an inline LIMIT plan node to this index scan plan node
LimitPlanNode lpn = new LimitPlanNode();
lpn.setLimit(1);
lpn.setOffset(0);
ispn.addInlinePlanNode(lpn);
// |__LimitPlanNode
if (sortDirection == SortDirectionType.DESC && !ispn.getSearchKeyExpressions().isEmpty() && exprs.isEmpty() && ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).isEmpty()) {
AbstractExpression newPredicate = new ComparisonExpression();
if (ispn.getLookupType() == IndexLookupType.GT)
newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHAN);
if (ispn.getLookupType() == IndexLookupType.GTE)
newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHANOREQUALTO);
newPredicate.setRight(ispn.getSearchKeyExpressions().get(0));
newPredicate.setLeft(aggExpr);
newPredicate.setValueType(aggExpr.getValueType());
ispn.clearSearchKeyExpression();
aggplan.setPrePredicate(newPredicate);
}
}
return plan;
}
Aggregations