Examples with ONeedRetryException com.orientechnologies.common.concur.ONeedRetryException used on opensource projects

Example 11 with ONeedRetryException

use of com.orientechnologies.common.concur.ONeedRetryException in project orientdb by orientechnologies.

the class ODistributedStorage method updateRecord.

@Override
public OStorageOperationResult<Integer> updateRecord(final ORecordId iRecordId, final boolean updateContent, final byte[] iContent, final int iVersion, final byte iRecordType, final int iMode, final ORecordCallback<Integer> iCallback) {
    resetLastValidBackup();
    if (OScenarioThreadLocal.INSTANCE.isRunModeDistributed()) {
        // ALREADY DISTRIBUTED
        return wrapped.updateRecord(iRecordId, updateContent, iContent, iVersion, iRecordType, iMode, iCallback);
    }
    checkLocalNodeIsAvailable();
    final ODistributedConfiguration dbCfg = distributedConfiguration;
    final String clusterName = getClusterNameByRID(iRecordId);
    final String localNodeName = dManager.getLocalNodeName();
    checkWriteQuorum(dbCfg, clusterName, localNodeName);
    try {
        checkNodeIsMaster(localNodeName, dbCfg);
        final List<String> nodes = dbCfg.getServers(clusterName, null);
        if (nodes.isEmpty())
            // NO REPLICATION: EXECUTE IT LOCALLY
            return wrapped.updateRecord(iRecordId, updateContent, iContent, iVersion, iRecordType, iMode, iCallback);
        final Set<String> clusterNames = Collections.singleton(clusterName);
        Boolean executionModeSynch = dbCfg.isExecutionModeSynchronous(clusterName);
        if (executionModeSynch == null)
            executionModeSynch = iMode == 0;
        final boolean syncMode = executionModeSynch;
        return (OStorageOperationResult<Integer>) executeRecordOperationInLock(syncMode, iRecordId, new OCallable<Object, OCallable<Void, ODistributedRequestId>>() {

            @Override
            public Object call(OCallable<Void, ODistributedRequestId> unlockCallback) {
                final OUpdateRecordTask task = new OUpdateRecordTask(iRecordId, iContent, iVersion, iRecordType);
                final OStorageOperationResult<Integer> localResult;
                final boolean executedLocally = nodes.contains(localNodeName);
                if (executedLocally) {
                    // EXECUTE ON LOCAL NODE FIRST
                    try {
                        // LOAD CURRENT RECORD
                        task.checkRecordExists();
                        localResult = (OStorageOperationResult<Integer>) OScenarioThreadLocal.executeAsDistributed(new Callable() {

                            @Override
                            public Object call() throws Exception {
                                task.setLastLSN(wrapped.getLSN());
                                return wrapped.updateRecord(iRecordId, updateContent, iContent, iVersion, iRecordType, iMode, iCallback);
                            }
                        });
                    } catch (RuntimeException e) {
                        throw e;
                    } catch (Exception e) {
                        throw OException.wrapException(new ODistributedException("Cannot delete record " + iRecordId), e);
                    }
                    nodes.remove(localNodeName);
                } else
                    localResult = null;
                if (nodes.isEmpty()) {
                    unlockCallback.call(null);
                    if (!executedLocally)
                        throw new ODistributedException("Cannot execute distributed update on record " + iRecordId + " because no nodes are available");
                } else {
                    final Integer localResultPayload = localResult != null ? localResult.getResult() : null;
                    if (syncMode || localResult == null) {
                        // REPLICATE IT
                        try {
                            final ODistributedResponse dResponse = dManager.sendRequest(getName(), clusterNames, nodes, task, dManager.getNextMessageIdCounter(), EXECUTION_MODE.RESPONSE, localResultPayload, unlockCallback);
                            final Object payload = dResponse.getPayload();
                            if (payload instanceof Exception) {
                                if (payload instanceof ORecordNotFoundException) {
                                    // REPAIR THE RECORD IMMEDIATELY
                                    localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord((ORecordId) ((ORecordNotFoundException) payload).getRid());
                                }
                                executeUndoOnLocalServer(dResponse.getRequestId(), task);
                                if (payload instanceof ONeedRetryException)
                                    throw (ONeedRetryException) payload;
                                throw OException.wrapException(new ODistributedException("Error on execution distributed update record"), (Exception) payload);
                            }
                            // UPDATE LOCALLY
                            return new OStorageOperationResult<Integer>((Integer) payload);
                        } catch (RuntimeException e) {
                            executeUndoOnLocalServer(null, task);
                            throw e;
                        } catch (Exception e) {
                            executeUndoOnLocalServer(null, task);
                            ODatabaseException.wrapException(new ODistributedException("Cannot execute distributed update record"), e);
                        }
                    }
                    // ASYNCHRONOUS CALL: EXECUTE LOCALLY AND THEN DISTRIBUTE
                    asynchronousExecution(new OAsynchDistributedOperation(getName(), Collections.singleton(clusterName), nodes, task, dManager.getNextMessageIdCounter(), localResultPayload, unlockCallback, null));
                }
                return localResult;
            }
        });
    } catch (ONeedRetryException e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(iRecordId);
        // PASS THROUGH
        throw e;
    } catch (HazelcastInstanceNotActiveException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (HazelcastException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (Exception e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(iRecordId);
        handleDistributedException("Cannot route UPDATE_RECORD operation for %s to the distributed node", e, iRecordId);
        // UNREACHABLE
        return null;
    }
}

Example 12 with ONeedRetryException

use of com.orientechnologies.common.concur.ONeedRetryException in project orientdb by orientechnologies.

the class OFunction method execute.

public Object execute(final Map<Object, Object> iArgs) {
    final long start = Orient.instance().getProfiler().startChrono();
    Object result;
    while (true) {
        try {
            if (callback != null)
                return callback.call(iArgs);
            final OCommandExecutorScript command = new OCommandExecutorScript();
            command.parse(new OCommandScript(getLanguage(), getCode()));
            result = command.execute(iArgs);
            break;
        } catch (ONeedRetryException e) {
            continue;
        } catch (ORetryQueryException e) {
            continue;
        }
    }
    if (Orient.instance().getProfiler().isRecording())
        Orient.instance().getProfiler().stopChrono("db." + ODatabaseRecordThreadLocal.INSTANCE.get().getName() + ".function.execute", "Time to execute a function", start, "db.*.function.execute");
    return result;
}

Example 13 with ONeedRetryException

use of com.orientechnologies.common.concur.ONeedRetryException in project orientdb by orientechnologies.

the class ODistributedStorage method createRecord.

public OStorageOperationResult<OPhysicalPosition> createRecord(final ORecordId iRecordId, final byte[] iContent, final int iRecordVersion, final byte iRecordType, final int iMode, final ORecordCallback<Long> iCallback) {
    resetLastValidBackup();
    if (OScenarioThreadLocal.INSTANCE.isRunModeDistributed()) {
        // ALREADY DISTRIBUTED
        return wrapped.createRecord(iRecordId, iContent, iRecordVersion, iRecordType, iMode, iCallback);
    }
    checkLocalNodeIsAvailable();
    checkClusterRebalanceIsNotRunning();
    final String localNodeName = dManager.getLocalNodeName();
    final ODistributedConfiguration dbCfg = distributedConfiguration;
    // ASSIGN DESTINATION NODE
    final int clusterId = iRecordId.getClusterId();
    if (clusterId == ORID.CLUSTER_ID_INVALID)
        throw new IllegalArgumentException("Cluster not valid");
    checkNodeIsMaster(localNodeName, dbCfg);
    final String clusterName = getClusterNameByRID(iRecordId);
    checkWriteQuorum(dbCfg, clusterName, localNodeName);
    try {
        ODocument documentForClusterSelection = iRecordId.getRecord();
        if (documentForClusterSelection == null) {
            // DOCUMENT NOT FOUND: BUILD A TEMPORARY ONE
            documentForClusterSelection = (ODocument) ORecordInternal.fill(new ODocument(), iRecordId, iRecordVersion, iContent, false);
        }
        checkForCluster(documentForClusterSelection, localNodeName, dbCfg);
        final List<String> servers = dbCfg.getServers(clusterName, null);
        if (servers.isEmpty())
            // NO NODES: EXECUTE LOCALLY ONLY
            return wrapped.createRecord(iRecordId, iContent, iRecordVersion, iRecordType, iMode, iCallback);
        final String finalClusterName = clusterName;
        final Set<String> clusterNames = Collections.singleton(finalClusterName);
        // REMOVE CURRENT NODE BECAUSE IT HAS BEEN ALREADY EXECUTED LOCALLY
        servers.remove(localNodeName);
        Boolean executionModeSynch = dbCfg.isExecutionModeSynchronous(finalClusterName);
        if (executionModeSynch == null)
            executionModeSynch = iMode == 0;
        final boolean syncMode = executionModeSynch;
        // IN ANY CASE EXECUTE LOCALLY AND THEN DISTRIBUTE
        return (OStorageOperationResult<OPhysicalPosition>) executeRecordOperationInLock(syncMode, iRecordId, new OCallable<Object, OCallable<Void, ODistributedRequestId>>() {

            @Override
            public Object call(OCallable<Void, ODistributedRequestId> unlockCallback) {
                final OStorageOperationResult<OPhysicalPosition> localResult;
                localResult = wrapped.createRecord(iRecordId, iContent, iRecordVersion, iRecordType, iMode, iCallback);
                // UPDATE RID WITH NEW POSITION
                iRecordId.setClusterPosition(localResult.getResult().clusterPosition);
                final OPlaceholder localPlaceholder = new OPlaceholder(iRecordId, localResult.getResult().recordVersion);
                final OCreateRecordTask task = new OCreateRecordTask(iRecordId, iContent, iRecordVersion, iRecordType);
                task.setLastLSN(wrapped.getLSN());
                if (!servers.isEmpty()) {
                    if (syncMode) {
                        // SYNCHRONOUS CALL: REPLICATE IT
                        try {
                            final ODistributedResponse dResponse = dManager.sendRequest(getName(), clusterNames, servers, task, dManager.getNextMessageIdCounter(), EXECUTION_MODE.RESPONSE, localPlaceholder, unlockCallback);
                            final Object payload = dResponse.getPayload();
                            if (payload != null) {
                                if (payload instanceof Exception) {
                                    executeUndoOnLocalServer(dResponse.getRequestId(), task);
                                    if (payload instanceof ONeedRetryException)
                                        throw (ONeedRetryException) payload;
                                    throw OException.wrapException(new ODistributedException("Error on execution distributed create record"), (Exception) payload);
                                }
                                // COPY THE CLUSTER POS -> RID
                                final OPlaceholder masterPlaceholder = (OPlaceholder) payload;
                                iRecordId.copyFrom(masterPlaceholder.getIdentity());
                                return new OStorageOperationResult<OPhysicalPosition>(new OPhysicalPosition(masterPlaceholder.getIdentity().getClusterPosition(), masterPlaceholder.getVersion()));
                            }
                        } catch (RuntimeException e) {
                            executeUndoOnLocalServer(null, task);
                            throw e;
                        } catch (Exception e) {
                            executeUndoOnLocalServer(null, task);
                            ODatabaseException.wrapException(new ODistributedException("Cannot execute distributed create record"), e);
                        }
                    } else {
                        // ASYNCHRONOUSLY REPLICATE IT TO ALL THE OTHER NODES
                        asynchronousExecution(new OAsynchDistributedOperation(getName(), Collections.singleton(finalClusterName), servers, task, dManager.getNextMessageIdCounter(), localPlaceholder, unlockCallback, null));
                    }
                } else
                    unlockCallback.call(null);
                return localResult;
            }
        });
    } catch (ODistributedRecordLockedException e) {
        // PASS THROUGH
        throw e;
    } catch (ONeedRetryException e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(iRecordId);
        final ORecordId lockEntireCluster = iRecordId.copy();
        lockEntireCluster.setClusterPosition(-1);
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(lockEntireCluster);
        // PASS THROUGH
        throw e;
    } catch (HazelcastInstanceNotActiveException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (HazelcastException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (Exception e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(iRecordId);
        final ORecordId lockEntireCluster = iRecordId.copy();
        lockEntireCluster.setClusterPosition(-1);
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(lockEntireCluster);
        handleDistributedException("Cannot route create record operation for %s to the distributed node", e, iRecordId);
        // UNREACHABLE
        return null;
    }
}

Example 14 with ONeedRetryException

use of com.orientechnologies.common.concur.ONeedRetryException in project orientdb by orientechnologies.

the class ODistributedStorage method command.

public Object command(final OCommandRequestText iCommand) {
    List<String> servers = (List<String>) iCommand.getContext().getVariable("servers");
    if (servers == null) {
        servers = new ArrayList<String>();
        iCommand.getContext().setVariable("servers", servers);
    }
    final String localNodeName = dManager.getLocalNodeName();
    servers.add(localNodeName);
    if (OScenarioThreadLocal.INSTANCE.isRunModeDistributed())
        // ALREADY DISTRIBUTED
        return wrapped.command(iCommand);
    final ODistributedConfiguration dbCfg = distributedConfiguration;
    if (!dbCfg.isReplicationActive(null, localNodeName))
        // DON'T REPLICATE
        return wrapped.command(iCommand);
    final OCommandExecutor executor = OCommandManager.instance().getExecutor(iCommand);
    executor.setProgressListener(iCommand.getProgressListener());
    executor.parse(iCommand);
    final OCommandExecutor exec = executor instanceof OCommandExecutorSQLDelegate ? ((OCommandExecutorSQLDelegate) executor).getDelegate() : executor;
    if (exec.isIdempotent() && !dManager.isNodeAvailable(dManager.getLocalNodeName(), getName())) {
        // SPECIAL CASE: NODE IS OFFLINE AND THE COMMAND IS IDEMPOTENT, EXECUTE IT LOCALLY ONLY
        ODistributedServerLog.warn(this, dManager.getLocalNodeName(), null, ODistributedServerLog.DIRECTION.NONE, "Node '%s' is %s, the command '%s' against database '%s' will be executed only on local server with the possibility to have partial result", dManager.getLocalNodeName(), dManager.getDatabaseStatus(dManager.getLocalNodeName(), getName()), iCommand, wrapped.getName());
        return wrapped.command(iCommand);
    }
    checkLocalNodeIsAvailable();
    if (!exec.isIdempotent())
        checkNodeIsMaster(localNodeName, dbCfg);
    try {
        Object result = null;
        OCommandDistributedReplicateRequest.DISTRIBUTED_EXECUTION_MODE executionMode = OCommandDistributedReplicateRequest.DISTRIBUTED_EXECUTION_MODE.LOCAL;
        OCommandDistributedReplicateRequest.DISTRIBUTED_RESULT_MGMT resultMgmt = OCommandDistributedReplicateRequest.DISTRIBUTED_RESULT_MGMT.CHECK_FOR_EQUALS;
        boolean executeOnLocalNodeFirst = true;
        if (OScenarioThreadLocal.INSTANCE.getRunMode() != RUN_MODE.RUNNING_DISTRIBUTED) {
            if (exec instanceof OCommandDistributedReplicateRequest) {
                executionMode = ((OCommandDistributedReplicateRequest) exec).getDistributedExecutionMode();
                resultMgmt = ((OCommandDistributedReplicateRequest) exec).getDistributedResultManagement();
                executeOnLocalNodeFirst = ((OCommandDistributedReplicateRequest) exec).isDistributedExecutingOnLocalNodeFirst();
            }
        }
        switch(executionMode) {
            case LOCAL:
                // CALL IN DEFAULT MODE TO LET OWN COMMAND TO REDISTRIBUTE CHANGES (LIKE INSERT)
                return wrapped.command(iCommand);
            case REPLICATE:
                // REPLICATE IT, GET ALL THE INVOLVED NODES
                final Collection<String> involvedClusters = exec.getInvolvedClusters();
                if (resultMgmt == OCommandDistributedReplicateRequest.DISTRIBUTED_RESULT_MGMT.MERGE) {
                    if (!exec.isIdempotent() && dbCfg.isSharded())
                        throw new ODistributedOperationException("Cannot distribute the command '" + iCommand.getText() + "' because it is not idempotent and a map-reduce has been requested");
                    final Map<String, Collection<String>> nodeClusterMap = dbCfg.getServerClusterMap(involvedClusters, localNodeName, exec.isIdempotent());
                    final Map<String, Object> results;
                    if (exec.isIdempotent() && nodeClusterMap.size() == 1 && nodeClusterMap.keySet().iterator().next().equals(localNodeName)) {
                        // LOCAL NODE, AVOID TO DISTRIBUTE IT
                        // CALL IN DEFAULT MODE TO LET OWN COMMAND TO REDISTRIBUTE CHANGES (LIKE INSERT)
                        result = wrapped.command(iCommand);
                        results = new HashMap<String, Object>(1);
                        results.put(localNodeName, result);
                    } else {
                        // SELECT: SPLIT CLASSES/CLUSTER IF ANY
                        results = executeOnServers(iCommand, exec, involvedClusters, nodeClusterMap);
                    }
                    final OCommandExecutorSQLSelect select = exec instanceof OCommandExecutorSQLSelect ? (OCommandExecutorSQLSelect) exec : null;
                    if (select != null && select.isAnyFunctionAggregates() && !select.hasGroupBy()) {
                        result = mergeResultByAggregation(select, results);
                    } else {
                        // MIX & FILTER RESULT SET AVOIDING DUPLICATES
                        // TODO: ONCE OPTIMIZED (SEE ABOVE) AVOID TO FILTER HERE
                        result = exec.mergeResults(results);
                    }
                    if (result instanceof Throwable && results.containsKey(localNodeName))
                        undoCommandOnLocalServer(iCommand);
                } else {
                    final OAbstractCommandTask task = iCommand instanceof OCommandScript ? new OScriptTask(iCommand) : new OSQLCommandTask(iCommand, new HashSet<String>());
                    task.setResultStrategy(OAbstractRemoteTask.RESULT_STRATEGY.ANY);
                    final Set<String> nodes = dbCfg.getServers(involvedClusters);
                    if (iCommand instanceof ODistributedCommand)
                        nodes.removeAll(((ODistributedCommand) iCommand).nodesToExclude());
                    if (executeOnlyLocally(localNodeName, dbCfg, exec, involvedClusters, nodes))
                        // CALL IN DEFAULT MODE TO LET OWN COMMAND TO REDISTRIBUTE CHANGES (LIKE INSERT)
                        return wrapped.command(iCommand);
                    final Object localResult;
                    final boolean executedLocally = executeOnLocalNodeFirst && nodes.contains(localNodeName);
                    if (exec.involveSchema())
                        // EXECUTE THE COMMAND IN LOCK
                        result = dManager.executeInDistributedDatabaseLock(getName(), 0, dManager.getDatabaseConfiguration(getName()).modify(), new OCallable<Object, OModifiableDistributedConfiguration>() {

                            @Override
                            public Object call(OModifiableDistributedConfiguration iArgument) {
                                return executeCommand(iCommand, localNodeName, involvedClusters, task, nodes, executedLocally);
                            }
                        });
                    else
                        result = executeCommand(iCommand, localNodeName, involvedClusters, task, nodes, executedLocally);
                }
                if (exec.involveSchema())
                    // UPDATE THE SCHEMA
                    dManager.propagateSchemaChanges(ODatabaseRecordThreadLocal.INSTANCE.get());
                break;
        }
        if (result instanceof ONeedRetryException)
            throw (ONeedRetryException) result;
        else if (result instanceof RuntimeException)
            throw (RuntimeException) result;
        else if (result instanceof Exception)
            throw OException.wrapException(new ODistributedException("Error on execution distributed COMMAND"), (Exception) result);
        return result;
    } catch (OConcurrentModificationException e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord((ORecordId) e.getRid());
        throw e;
    } catch (ONeedRetryException e) {
        // PASS THROUGH
        throw e;
    } catch (HazelcastInstanceNotActiveException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (HazelcastException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (Exception e) {
        handleDistributedException("Cannot route COMMAND operation to the distributed node", e);
        // UNREACHABLE
        return null;
    }
}

Example 15 with ONeedRetryException

use of com.orientechnologies.common.concur.ONeedRetryException in project orientdb by orientechnologies.

the class ODistributedStorage method deleteRecord.

@Override
public OStorageOperationResult<Boolean> deleteRecord(final ORecordId iRecordId, final int iVersion, final int iMode, final ORecordCallback<Boolean> iCallback) {
    resetLastValidBackup();
    if (OScenarioThreadLocal.INSTANCE.isRunModeDistributed()) {
        // ALREADY DISTRIBUTED
        return wrapped.deleteRecord(iRecordId, iVersion, iMode, iCallback);
    }
    checkLocalNodeIsAvailable();
    final String clusterName = getClusterNameByRID(iRecordId);
    final ODistributedConfiguration dbCfg = distributedConfiguration;
    final String localNodeName = dManager.getLocalNodeName();
    checkWriteQuorum(dbCfg, clusterName, localNodeName);
    try {
        checkNodeIsMaster(localNodeName, dbCfg);
        final List<String> nodes = dbCfg.getServers(clusterName, null);
        if (nodes.isEmpty())
            // NO NODES: EXECUTE LOCALLY ONLY
            return wrapped.deleteRecord(iRecordId, iVersion, iMode, iCallback);
        final Set<String> clusterNames = Collections.singleton(clusterName);
        Boolean executionModeSynch = dbCfg.isExecutionModeSynchronous(clusterName);
        if (executionModeSynch == null)
            executionModeSynch = iMode == 0;
        final boolean syncMode = executionModeSynch;
        return (OStorageOperationResult<Boolean>) executeRecordOperationInLock(syncMode, iRecordId, new OCallable<Object, OCallable<Void, ODistributedRequestId>>() {

            @Override
            public Object call(OCallable<Void, ODistributedRequestId> unlockCallback) {
                final ODeleteRecordTask task = new ODeleteRecordTask(iRecordId, iVersion);
                final OStorageOperationResult<Boolean> localResult;
                final boolean executedLocally = nodes.contains(localNodeName);
                if (executedLocally) {
                    // EXECUTE ON LOCAL NODE FIRST
                    try {
                        // LOAD CURRENT RECORD
                        task.checkRecordExists();
                        localResult = (OStorageOperationResult<Boolean>) OScenarioThreadLocal.executeAsDistributed(new Callable() {

                            @Override
                            public Object call() throws Exception {
                                task.setLastLSN(wrapped.getLSN());
                                return wrapped.deleteRecord(iRecordId, iVersion, iMode, iCallback);
                            }
                        });
                    } catch (RuntimeException e) {
                        throw e;
                    } catch (Exception e) {
                        throw OException.wrapException(new ODistributedException("Cannot delete record " + iRecordId), e);
                    }
                    nodes.remove(localNodeName);
                } else
                    localResult = null;
                if (nodes.isEmpty()) {
                    unlockCallback.call(null);
                    if (!executedLocally)
                        throw new ODistributedException("Cannot execute distributed delete on record " + iRecordId + " because no nodes are available");
                } else {
                    final Boolean localResultPayload = localResult != null ? localResult.getResult() : null;
                    if (syncMode || localResult == null) {
                        // REPLICATE IT
                        try {
                            final ODistributedResponse dResponse = dManager.sendRequest(getName(), clusterNames, nodes, task, dManager.getNextMessageIdCounter(), EXECUTION_MODE.RESPONSE, localResultPayload, unlockCallback);
                            final Object payload = dResponse.getPayload();
                            if (payload instanceof Exception) {
                                if (payload instanceof ORecordNotFoundException) {
                                    // REPAIR THE RECORD IMMEDIATELY
                                    localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord((ORecordId) ((ORecordNotFoundException) payload).getRid());
                                }
                                executeUndoOnLocalServer(dResponse.getRequestId(), task);
                                if (payload instanceof ONeedRetryException)
                                    throw (ONeedRetryException) payload;
                                throw OException.wrapException(new ODistributedException("Error on execution distributed delete record"), (Exception) payload);
                            }
                            return new OStorageOperationResult<Boolean>(true);
                        } catch (RuntimeException e) {
                            executeUndoOnLocalServer(null, task);
                            throw e;
                        } catch (Exception e) {
                            executeUndoOnLocalServer(null, task);
                            ODatabaseException.wrapException(new ODistributedException("Cannot execute distributed delete record"), e);
                        }
                    }
                    // ASYNCHRONOUS CALL: EXECUTE LOCALLY AND THEN DISTRIBUTE
                    if (!nodes.isEmpty())
                        asynchronousExecution(new OAsynchDistributedOperation(getName(), Collections.singleton(clusterName), nodes, task, dManager.getNextMessageIdCounter(), localResultPayload, unlockCallback, null));
                }
                return localResult;
            }
        });
    } catch (ONeedRetryException e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(iRecordId);
        // PASS THROUGH
        throw e;
    } catch (HazelcastInstanceNotActiveException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (HazelcastException e) {
        throw new OOfflineNodeException("Hazelcast instance is not available");
    } catch (Exception e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord(iRecordId);
        handleDistributedException("Cannot route DELETE_RECORD operation for %s to the distributed node", e, iRecordId);
        // UNREACHABLE
        return null;
    }
}