Examples with ORemoteTask com.orientechnologies.orient.server.distributed.task.ORemoteTask used on opensource projects

Example 1 with ORemoteTask

use of com.orientechnologies.orient.server.distributed.task.ORemoteTask in project orientdb by orientechnologies.

the class ODistributedDatabaseImpl method send2Nodes.

@Override
public ODistributedResponse send2Nodes(final ODistributedRequest iRequest, final Collection<String> iClusterNames, Collection<String> iNodes, final ODistributedRequest.EXECUTION_MODE iExecutionMode, final Object localResult, final OCallable<Void, ODistributedRequestId> iAfterSentCallback) {
    boolean afterSendCallBackCalled = false;
    try {
        checkForServerOnline(iRequest);
        final String databaseName = iRequest.getDatabaseName();
        if (iNodes.isEmpty()) {
            ODistributedServerLog.error(this, localNodeName, null, DIRECTION.OUT, "No nodes configured for database '%s' request: %s", databaseName, iRequest);
            throw new ODistributedException("No nodes configured for partition '" + databaseName + "' request: " + iRequest);
        }
        final ODistributedConfiguration cfg = manager.getDatabaseConfiguration(databaseName);
        final ORemoteTask task = iRequest.getTask();
        final boolean checkNodesAreOnline = task.isNodeOnlineRequired();
        final Set<String> nodesConcurToTheQuorum = manager.getDistributedStrategy().getNodesConcurInQuorum(manager, cfg, iRequest, iNodes, localResult);
        // AFTER COMPUTED THE QUORUM, REMOVE THE OFFLINE NODES TO HAVE THE LIST OF REAL AVAILABLE NODES
        final int availableNodes = checkNodesAreOnline ? manager.getNodesWithStatus(iNodes, databaseName, ODistributedServerManager.DB_STATUS.ONLINE, ODistributedServerManager.DB_STATUS.BACKUP, ODistributedServerManager.DB_STATUS.SYNCHRONIZING) : iNodes.size();
        final int expectedResponses = localResult != null ? availableNodes + 1 : availableNodes;
        final int quorum = calculateQuorum(task.getQuorumType(), iClusterNames, cfg, expectedResponses, nodesConcurToTheQuorum.size(), checkNodesAreOnline, localNodeName);
        final boolean groupByResponse = task.getResultStrategy() != OAbstractRemoteTask.RESULT_STRATEGY.UNION;
        final boolean waitLocalNode = waitForLocalNode(cfg, iClusterNames, iNodes);
        // CREATE THE RESPONSE MANAGER
        final ODistributedResponseManager currentResponseMgr = new ODistributedResponseManager(manager, iRequest, iNodes, nodesConcurToTheQuorum, expectedResponses, quorum, waitLocalNode, task.getSynchronousTimeout(expectedResponses), task.getTotalTimeout(availableNodes), groupByResponse);
        if (localResult != null)
            // COLLECT LOCAL RESULT
            currentResponseMgr.setLocalResult(localNodeName, localResult);
        // SORT THE NODE TO GUARANTEE THE SAME ORDER OF DELIVERY
        if (!(iNodes instanceof List))
            iNodes = new ArrayList<String>(iNodes);
        if (iNodes.size() > 1)
            Collections.sort((List<String>) iNodes);
        msgService.registerRequest(iRequest.getId().getMessageId(), currentResponseMgr);
        if (ODistributedServerLog.isDebugEnabled())
            ODistributedServerLog.debug(this, localNodeName, iNodes.toString(), DIRECTION.OUT, "Sending request %s...", iRequest);
        for (String node : iNodes) {
            // CATCH ANY EXCEPTION LOG IT AND IGNORE TO CONTINUE SENDING REQUESTS TO OTHER NODES
            try {
                final ORemoteServerController remoteServer = manager.getRemoteServer(node);
                remoteServer.sendRequest(iRequest);
            } catch (Throwable e) {
                currentResponseMgr.removeServerBecauseUnreachable(node);
                String reason = e.getMessage();
                if (e instanceof ODistributedException && e.getCause() instanceof IOException) {
                    // CONNECTION ERROR: REMOVE THE CONNECTION
                    reason = e.getCause().getMessage();
                    manager.closeRemoteServer(node);
                } else if (e instanceof OSecurityAccessException) {
                    // THE CONNECTION COULD BE STALE, CREATE A NEW ONE AND RETRY
                    manager.closeRemoteServer(node);
                    try {
                        final ORemoteServerController remoteServer = manager.getRemoteServer(node);
                        remoteServer.sendRequest(iRequest);
                        continue;
                    } catch (Throwable ex) {
                    // IGNORE IT BECAUSE MANAGED BELOW
                    }
                }
                if (!manager.isNodeAvailable(node))
                    // NODE IS NOT AVAILABLE
                    ODistributedServerLog.debug(this, localNodeName, node, ODistributedServerLog.DIRECTION.OUT, "Error on sending distributed request %s. The target node is not available. Active nodes: %s", e, iRequest, manager.getAvailableNodeNames(databaseName));
                else
                    ODistributedServerLog.error(this, localNodeName, node, ODistributedServerLog.DIRECTION.OUT, "Error on sending distributed request %s (err=%s). Active nodes: %s", iRequest, reason, manager.getAvailableNodeNames(databaseName));
            }
        }
        if (ODistributedServerLog.isDebugEnabled())
            ODistributedServerLog.debug(this, localNodeName, iNodes.toString(), DIRECTION.OUT, "Sent request %s", iRequest);
        totalSentRequests.incrementAndGet();
        afterSendCallBackCalled = true;
        if (iAfterSentCallback != null)
            iAfterSentCallback.call(iRequest.getId());
        if (iExecutionMode == ODistributedRequest.EXECUTION_MODE.RESPONSE)
            return waitForResponse(iRequest, currentResponseMgr);
        return null;
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw OException.wrapException(new ODistributedException("Error on executing distributed request (" + iRequest + ") against database '" + databaseName + (iClusterNames != null ? "." + iClusterNames : "") + "' to nodes " + iNodes), e);
    } finally {
        if (iAfterSentCallback != null && !afterSendCallBackCalled)
            iAfterSentCallback.call(iRequest.getId());
    }
}

Example 2 with ORemoteTask

use of com.orientechnologies.orient.server.distributed.task.ORemoteTask in project orientdb by orientechnologies.

the class ODistributedDatabaseImpl method processRequest.

/**
   * Distributed requests against the available workers by using one queue per worker. This guarantee the sequence of the operations
   * against the same record cluster.
   */
public void processRequest(final ODistributedRequest request) {
    if (!running)
        // DISCARD IT
        return;
    final ORemoteTask task = request.getTask();
    waitIsReady(task);
    if (!running)
        // DISCARD IT
        return;
    totalReceivedRequests.incrementAndGet();
    // final ODistributedMomentum lastMomentum = filterByMomentum.get();
    // if (lastMomentum != null && task instanceof OAbstractReplicatedTask) {
    // final OLogSequenceNumber taskLastLSN = ((OAbstractReplicatedTask) task).getLastLSN();
    //
    // final String sourceServer = manager.getNodeNameById(request.getId().getNodeId());
    // final OLogSequenceNumber lastLSNFromMomentum = lastMomentum.getLSN(sourceServer);
    //
    // if (taskLastLSN != null && lastLSNFromMomentum != null && taskLastLSN.compareTo(lastLSNFromMomentum) < 0) {
    // // SKIP REQUEST BECAUSE CONTAINS AN OLD LSN
    // final String msg = String.format("Skipped request %s on database '%s' because %s < current %s", request, databaseName,
    // taskLastLSN, lastLSNFromMomentum);
    // ODistributedServerLog.info(this, localNodeName, null, DIRECTION.NONE, msg);
    // ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedException(msg));
    // return;
    // }
    // }
    final int[] partitionKeys = task.getPartitionKey();
    if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, task.getNodeSource(), DIRECTION.IN, "Request %s on database '%s' partitionKeys=%s task=%s", request, databaseName, Arrays.toString(partitionKeys), task);
    if (partitionKeys.length > 1 || partitionKeys[0] == -1) {
        final Set<Integer> involvedWorkerQueues;
        if (partitionKeys.length > 1)
            involvedWorkerQueues = getInvolvedQueuesByPartitionKeys(partitionKeys);
        else
            // LOCK ALL THE QUEUES
            involvedWorkerQueues = ALL_QUEUES;
        // if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE, "Request %s on database '%s' involvedQueues=%s", request, databaseName, involvedWorkerQueues);
        if (involvedWorkerQueues.size() == 1)
            // JUST ONE QUEUE INVOLVED: PROCESS IT IMMEDIATELY
            processRequest(involvedWorkerQueues.iterator().next(), request);
        else {
            // INVOLVING MULTIPLE QUEUES
            // if (ODistributedServerLog.isDebugEnabled())
            ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE, "Request %s on database '%s' waiting for all the previous requests to be completed", request, databaseName);
            // WAIT ALL THE INVOLVED QUEUES ARE FREE AND SYNCHRONIZED
            final CountDownLatch syncLatch = new CountDownLatch(involvedWorkerQueues.size());
            final ODistributedRequest syncRequest = new ODistributedRequest(manager.getTaskFactory(), request.getId().getNodeId(), -1, databaseName, new OSynchronizedTaskWrapper(syncLatch));
            for (int queue : involvedWorkerQueues) workerThreads.get(queue).processRequest(syncRequest);
            long taskTimeout = task.getDistributedTimeout();
            try {
                if (taskTimeout <= 0)
                    syncLatch.await();
                else {
                    // WAIT FOR COMPLETION. THE TIMEOUT IS MANAGED IN SMALLER CYCLES TO PROPERLY RECOGNIZE WHEN THE DB IS REMOVED
                    final long start = System.currentTimeMillis();
                    final long cycleTimeout = Math.min(taskTimeout, 2000);
                    boolean locked = false;
                    do {
                        if (syncLatch.await(cycleTimeout, TimeUnit.MILLISECONDS)) {
                            // DONE
                            locked = true;
                            break;
                        }
                        if (this.workerThreads.size() == 0)
                            // DATABASE WAS SHUTDOWN
                            break;
                    } while (System.currentTimeMillis() - start < taskTimeout);
                    if (!locked) {
                        final String msg = String.format("Cannot execute distributed request (%s) because all worker threads (%d) are busy (pending=%d)", request, workerThreads.size(), syncLatch.getCount());
                        ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedOperationException(msg));
                        return;
                    }
                }
            } catch (InterruptedException e) {
                // IGNORE
                Thread.currentThread().interrupt();
                final String msg = String.format("Cannot execute distributed request (%s) because all worker threads (%d) are busy", request, workerThreads.size());
                ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedOperationException(msg));
                return;
            }
            // PUT THE TASK TO EXECUTE ONLY IN THE FIRST QUEUE AND PUT WAIT-FOR TASKS IN THE OTHERS. WHEN THE REAL TASK IS EXECUTED,
            // ALL THE OTHER TASKS WILL RETURN, SO THE QUEUES WILL BE BUSY DURING THE EXECUTION OF THE TASK. THIS AVOID CONCURRENT
            // EXECUTION FOR THE SAME PARTITION
            final CountDownLatch queueLatch = new CountDownLatch(1);
            int i = 0;
            for (int queue : involvedWorkerQueues) {
                final ODistributedRequest req;
                if (i++ == 0) {
                    // USE THE FIRST QUEUE TO PROCESS THE REQUEST
                    final String senderNodeName = manager.getNodeNameById(request.getId().getNodeId());
                    request.setTask(new OSynchronizedTaskWrapper(queueLatch, senderNodeName, task));
                    req = request;
                } else
                    req = new ODistributedRequest(manager.getTaskFactory(), request.getId().getNodeId(), -1, databaseName, new OWaitForTask(queueLatch));
                workerThreads.get(queue).processRequest(req);
            }
        }
    } else if (partitionKeys.length > 1 || partitionKeys[0] == -2) {
        // ANY PARTITION: USE THE FIRST EMPTY IF ANY, OTHERWISE THE FIRST IN THE LIST
        boolean found = false;
        for (ODistributedWorker q : workerThreads) {
            if (q.isWaitingForNextRequest() && q.localQueue.isEmpty()) {
                q.processRequest(request);
                found = true;
                break;
            }
        }
        if (!found)
            // ALL THE THREADS ARE BUSY, SELECT THE FIRST EMPTY ONE
            for (ODistributedWorker q : workerThreads) {
                if (q.localQueue.isEmpty()) {
                    q.processRequest(request);
                    found = true;
                    break;
                }
            }
        if (!found)
            // EXEC ON THE FIRST QUEUE
            workerThreads.get(0).processRequest(request);
    } else {
        processRequest(partitionKeys[0], request);
    }
}

Example 3 with ORemoteTask

use of com.orientechnologies.orient.server.distributed.task.ORemoteTask in project orientdb by orientechnologies.

the class ODistributedResponseManager method fixNodesInConflict.

protected boolean fixNodesInConflict(final List<ODistributedResponse> bestResponsesGroup, final int conflicts) {
    // NO FIFTY/FIFTY CASE: FIX THE CONFLICTED NODES BY OVERWRITING THE RECORD WITH THE WINNER'S RESULT
    final ODistributedResponse goodResponse = bestResponsesGroup.get(0);
    if (goodResponse.getPayload() instanceof Throwable)
        return false;
    ODistributedServerLog.debug(this, dManager.getLocalNodeName(), null, DIRECTION.NONE, "Detected %d conflicts, but the quorum (%d) has been reached. Fixing remote records. Request (%s)", conflicts, quorum, request);
    for (List<ODistributedResponse> responseGroup : responseGroups) {
        if (responseGroup != bestResponsesGroup) {
            // CONFLICT GROUP: FIX THEM ONE BY ONE
            for (ODistributedResponse r : responseGroup) {
                if (r.getPayload() instanceof ODistributedRecordLockedException)
                    // NO FIX, THE RECORD WAS LOCKED
                    return false;
                final ORemoteTask fixTask = ((OAbstractReplicatedTask) request.getTask()).getFixTask(request, request.getTask(), r.getPayload(), goodResponse.getPayload(), r.getExecutorNodeName(), dManager);
                if (fixTask == null)
                    // FIX NOT AVAILABLE: UNDO THE ENTIRE OPERATION
                    return false;
                ODistributedServerLog.debug(this, dManager.getLocalNodeName(), r.getExecutorNodeName(), DIRECTION.OUT, "Sending fix message (%s) for response (%s) on request (%s) to be: %s", fixTask, r, request, goodResponse);
                dManager.sendRequest(request.getDatabaseName(), null, OMultiValue.getSingletonList(r.getExecutorNodeName()), fixTask, dManager.getNextMessageIdCounter(), ODistributedRequest.EXECUTION_MODE.NO_RESPONSE, null, null);
            }
        }
    }
    return true;
}

Example 4 with ORemoteTask

use of com.orientechnologies.orient.server.distributed.task.ORemoteTask in project orientdb by orientechnologies.

the class ODistributedWorker method onMessage.

/**
 * Executes the remote call on the local node and send back the result
 */
protected void onMessage(final ODistributedRequest iRequest) {
    String senderNodeName = null;
    for (int retry = 0; retry < 10; retry++) {
        senderNodeName = manager.getNodeNameById(iRequest.getId().getNodeId());
        if (senderNodeName != null)
            break;
        try {
            Thread.sleep(200);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ODistributedException("Execution has been interrupted");
        }
    }
    if (senderNodeName == null) {
        ODistributedServerLog.warn(this, localNodeName, senderNodeName, DIRECTION.IN, "Sender server id %d is not registered in the cluster configuration, discard the request: (%s) (worker=%d)", iRequest.getId().getNodeId(), iRequest, id);
        sendResponseBack(iRequest, new ODistributedException("Sender server id " + iRequest.getId().getNodeId() + " is not registered in the cluster configuration, discard the request"));
        return;
    }
    final ORemoteTask task = iRequest.getTask();
    if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, senderNodeName, DIRECTION.IN, "Received request: (%s) (worker=%d)", iRequest, id);
    // EXECUTE IT LOCALLY
    Object responsePayload = null;
    OSecurityUser origin = null;
    try {
        waitNodeIsOnline();
        distributed.waitIsReady(task);
        if (task.isUsingDatabase()) {
            initDatabaseInstance();
            if (database == null)
                throw new ODistributedOperationException("Error on executing remote request because the database '" + databaseName + "' is not available");
        }
        // reset to original user
        if (database != null) {
            database.activateOnCurrentThread();
            origin = database.getUser();
            try {
                if (iRequest.getUserRID() != null && iRequest.getUserRID().isValid() && (lastUser == null || !(lastUser.getIdentity()).equals(iRequest.getUserRID()))) {
                    lastUser = database.getMetadata().getSecurity().getUser(iRequest.getUserRID());
                    // set to new user
                    database.setUser(lastUser);
                } else
                    origin = null;
            } catch (Throwable ex) {
                OLogManager.instance().error(this, "Failed on user switching database. " + ex.getMessage());
            }
        }
        // EXECUTE THE TASK
        for (int retry = 1; running; ++retry) {
            responsePayload = manager.executeOnLocalNode(iRequest.getId(), iRequest.getTask(), database);
            if (responsePayload instanceof OModificationOperationProhibitedException) {
                // RETRY
                try {
                    ODistributedServerLog.info(this, localNodeName, senderNodeName, DIRECTION.IN, "Database is frozen, waiting and retrying. Request %s (retry=%d, worker=%d)", iRequest, retry, id);
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                }
            } else {
                // OPERATION EXECUTED (OK OR ERROR), NO RETRY NEEDED
                if (retry > 1)
                    ODistributedServerLog.info(this, localNodeName, senderNodeName, DIRECTION.IN, "Request %s succeed after retry=%d", iRequest, retry);
                break;
            }
        }
    } catch (RuntimeException e) {
        if (task.hasResponse())
            sendResponseBack(iRequest, e);
        throw e;
    } finally {
        if (database != null && !database.isClosed()) {
            database.activateOnCurrentThread();
            if (!database.isClosed()) {
                database.rollback();
                database.getLocalCache().clear();
                if (origin != null)
                    database.setUser(origin);
            }
        }
    }
    if (task.hasResponse())
        sendResponseBack(iRequest, responsePayload);
}

Example 5 with ORemoteTask

use of com.orientechnologies.orient.server.distributed.task.ORemoteTask in project orientdb by orientechnologies.

the class ODistributedDatabaseImpl method processRequest.

/**
 * Distributed requests against the available workers by using one queue per worker. This guarantee the sequence of the operations
 * against the same record cluster.
 */
public void processRequest(final ODistributedRequest request, final boolean waitForAcceptingRequests) {
    if (!running)
        // DISCARD IT
        return;
    final ORemoteTask task = request.getTask();
    if (waitForAcceptingRequests) {
        waitIsReady(task);
        if (!running)
            // DISCARD IT
            return;
    }
    totalReceivedRequests.incrementAndGet();
    // final ODistributedMomentum lastMomentum = filterByMomentum.get();
    // if (lastMomentum != null && task instanceof OAbstractReplicatedTask) {
    // final OLogSequenceNumber taskLastLSN = ((OAbstractReplicatedTask) task).getLastLSN();
    // 
    // final String sourceServer = manager.getNodeNameById(request.getId().getNodeId());
    // final OLogSequenceNumber lastLSNFromMomentum = lastMomentum.getLSN(sourceServer);
    // 
    // if (taskLastLSN != null && lastLSNFromMomentum != null && taskLastLSN.compareTo(lastLSNFromMomentum) < 0) {
    // // SKIP REQUEST BECAUSE CONTAINS AN OLD LSN
    // final String msg = String.format("Skipped request %s on database '%s' because %s < current %s", request, databaseName,
    // taskLastLSN, lastLSNFromMomentum);
    // ODistributedServerLog.info(this, localNodeName, null, DIRECTION.NONE, msg);
    // ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedException(msg));
    // return;
    // }
    // }
    final int[] partitionKeys = task.getPartitionKey();
    if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, task.getNodeSource(), DIRECTION.IN, "Request %s on database '%s' partitionKeys=%s task=%s", request, databaseName, Arrays.toString(partitionKeys), task);
    if (partitionKeys.length > 1 || partitionKeys[0] == -1) {
        final Set<Integer> involvedWorkerQueues;
        if (partitionKeys.length > 1)
            involvedWorkerQueues = getInvolvedQueuesByPartitionKeys(partitionKeys);
        else
            // LOCK ALL THE QUEUES
            involvedWorkerQueues = ALL_QUEUES;
        // if (ODistributedServerLog.isDebugEnabled())
        ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE, "Request %s on database '%s' involvedQueues=%s", request, databaseName, involvedWorkerQueues);
        if (involvedWorkerQueues.size() == 1)
            // JUST ONE QUEUE INVOLVED: PROCESS IT IMMEDIATELY
            processRequest(involvedWorkerQueues.iterator().next(), request);
        else {
            // INVOLVING MULTIPLE QUEUES
            // if (ODistributedServerLog.isDebugEnabled())
            ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE, "Request %s on database '%s' waiting for all the previous requests to be completed", request, databaseName);
            // WAIT ALL THE INVOLVED QUEUES ARE FREE AND SYNCHRONIZED
            final CountDownLatch syncLatch = new CountDownLatch(involvedWorkerQueues.size());
            final ODistributedRequest syncRequest = new ODistributedRequest(null, request.getId().getNodeId(), -1, databaseName, new OSynchronizedTaskWrapper(syncLatch));
            for (int queue : involvedWorkerQueues) workerThreads.get(queue).processRequest(syncRequest);
            long taskTimeout = task.getDistributedTimeout();
            try {
                if (taskTimeout <= 0)
                    syncLatch.await();
                else {
                    // WAIT FOR COMPLETION. THE TIMEOUT IS MANAGED IN SMALLER CYCLES TO PROPERLY RECOGNIZE WHEN THE DB IS REMOVED
                    final long start = System.currentTimeMillis();
                    final long cycleTimeout = Math.min(taskTimeout, 2000);
                    boolean locked = false;
                    do {
                        if (syncLatch.await(cycleTimeout, TimeUnit.MILLISECONDS)) {
                            // DONE
                            locked = true;
                            break;
                        }
                        if (this.workerThreads.size() == 0)
                            // DATABASE WAS SHUTDOWN
                            break;
                    } while (System.currentTimeMillis() - start < taskTimeout);
                    if (!locked) {
                        final String msg = String.format("Cannot execute distributed request (%s) because all worker threads (%d) are busy (pending=%d timeout=%d)", request, workerThreads.size(), syncLatch.getCount(), taskTimeout);
                        ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedOperationException(msg));
                        return;
                    }
                }
            } catch (InterruptedException e) {
                // IGNORE
                Thread.currentThread().interrupt();
                final String msg = String.format("Cannot execute distributed request (%s) because all worker threads (%d) are busy", request, workerThreads.size());
                ODistributedWorker.sendResponseBack(this, manager, request, new ODistributedOperationException(msg));
                return;
            }
            // PUT THE TASK TO EXECUTE ONLY IN THE FIRST QUEUE AND PUT WAIT-FOR TASKS IN THE OTHERS. WHEN THE REAL TASK IS EXECUTED,
            // ALL THE OTHER TASKS WILL RETURN, SO THE QUEUES WILL BE BUSY DURING THE EXECUTION OF THE TASK. THIS AVOID CONCURRENT
            // EXECUTION FOR THE SAME PARTITION
            final CountDownLatch queueLatch = new CountDownLatch(1);
            int i = 0;
            for (int queue : involvedWorkerQueues) {
                final ODistributedRequest req;
                if (i++ == 0) {
                    // USE THE FIRST QUEUE TO PROCESS THE REQUEST
                    final String senderNodeName = manager.getNodeNameById(request.getId().getNodeId());
                    request.setTask(new OSynchronizedTaskWrapper(queueLatch, senderNodeName, task));
                    req = request;
                } else
                    req = new ODistributedRequest(manager, request.getId().getNodeId(), -1, databaseName, new OWaitForTask(queueLatch));
                workerThreads.get(queue).processRequest(req);
            }
        }
    } else if (partitionKeys.length == 1 && partitionKeys[0] == -2) {
        // ANY PARTITION: USE THE FIRST EMPTY IF ANY, OTHERWISE THE FIRST IN THE LIST
        boolean found = false;
        for (ODistributedWorker q : workerThreads) {
            if (q.isWaitingForNextRequest() && q.localQueue.isEmpty()) {
                q.processRequest(request);
                found = true;
                break;
            }
        }
        if (!found)
            // ALL THE THREADS ARE BUSY, SELECT THE FIRST EMPTY ONE
            for (ODistributedWorker q : workerThreads) {
                if (q.localQueue.isEmpty()) {
                    q.processRequest(request);
                    found = true;
                    break;
                }
            }
        if (!found)
            // EXEC ON THE FIRST QUEUE
            workerThreads.get(0).processRequest(request);
    } else if (partitionKeys.length == 1 && partitionKeys[0] == -3) {
        // SERVICE - LOCK
        ODistributedServerLog.debug(this, localNodeName, request.getTask().getNodeSource(), DIRECTION.IN, "Request %s on database '%s' dispatched to the lock worker", request, databaseName);
        lockThread.processRequest(request);
    } else if (partitionKeys.length == 1 && partitionKeys[0] == -4) {
        // SERVICE - FAST_NOLOCK
        ODistributedServerLog.debug(this, localNodeName, request.getTask().getNodeSource(), DIRECTION.IN, "Request %s on database '%s' dispatched to the nowait worker", request, databaseName);
        unlockThread.processRequest(request);
    } else {
        processRequest(partitionKeys[0], request);
    }
}