Examples with ORecordId com.orientechnologies.orient.core.id.ORecordId used on opensource projects

Example 66 with ORecordId

use of com.orientechnologies.orient.core.id.ORecordId in project orientdb by orientechnologies.

the class ODistributedTransactionManager method acquireMultipleRecordLocks.

/**
   * Acquires lock in block by using an optimistic approach with retry & random delay. In case any record is locked, all the lock
   * acquired so far are released before to retry.
   *
   * @throws InterruptedException
   */
protected void acquireMultipleRecordLocks(final OTransaction iTx, final int maxAutoRetry, final int autoRetryDelay, final ODistributedStorageEventListener eventListener, final ODistributedTxContext reqContext) throws InterruptedException {
    final List<ORecordId> recordsToLock = new ArrayList<ORecordId>();
    for (ORecordOperation op : iTx.getAllRecordEntries()) {
        recordsToLock.add((ORecordId) op.record.getIdentity());
    }
    acquireMultipleRecordLocks(this, dManager, localDistributedDatabase, recordsToLock, maxAutoRetry, autoRetryDelay, eventListener, reqContext, -1);
}

Example 67 with ORecordId

use of com.orientechnologies.orient.core.id.ORecordId in project orientdb by orientechnologies.

the class ODistributedTransactionManager method checkForClusterIds.

protected void checkForClusterIds(final OTransaction iTx, final String localNodeName, final ODistributedConfiguration dbCfg) {
    for (ORecordOperation op : iTx.getAllRecordEntries()) {
        final ORecordId rid = (ORecordId) op.getRecord().getIdentity();
        switch(op.type) {
            case ORecordOperation.CREATED:
                final ORecordId newRid = rid.copy();
                if (rid.getClusterId() < 1) {
                    final String clusterName = ((OTransactionAbstract) iTx).getClusterName(op.getRecord());
                    if (clusterName != null) {
                        newRid.setClusterId(ODatabaseRecordThreadLocal.INSTANCE.get().getClusterIdByName(clusterName));
                        iTx.updateIdentityAfterCommit(rid, newRid);
                    }
                }
                if (storage.checkForCluster(op.getRecord(), localNodeName, dbCfg) != null)
                    iTx.updateIdentityAfterCommit(rid, newRid);
                break;
        }
    }
}

Example 68 with ORecordId

use of com.orientechnologies.orient.core.id.ORecordId in project orientdb by orientechnologies.

the class ODistributedTransactionManager method commit.

public List<ORecordOperation> commit(final ODatabaseDocumentTx database, final OTransaction iTx, final Runnable callback, final ODistributedStorageEventListener eventListener) {
    final String localNodeName = dManager.getLocalNodeName();
    try {
        OTransactionInternal.setStatus((OTransactionAbstract) iTx, OTransaction.TXSTATUS.BEGUN);
        final ODistributedConfiguration dbCfg = dManager.getDatabaseConfiguration(storage.getName());
        // CHECK THE LOCAL NODE IS THE OWNER OF THE CLUSTER IDS
        checkForClusterIds(iTx, localNodeName, dbCfg);
        // CREATE UNDO CONTENT FOR DISTRIBUTED 2-PHASE ROLLBACK
        final List<OAbstractRemoteTask> undoTasks = createUndoTasksFromTx(iTx);
        final int maxAutoRetry = OGlobalConfiguration.DISTRIBUTED_CONCURRENT_TX_MAX_AUTORETRY.getValueAsInteger();
        final int autoRetryDelay = OGlobalConfiguration.DISTRIBUTED_CONCURRENT_TX_AUTORETRY_DELAY.getValueAsInteger();
        Boolean executionModeSynch = dbCfg.isExecutionModeSynchronous(null);
        if (executionModeSynch == null)
            executionModeSynch = Boolean.TRUE;
        final boolean finalExecutionModeSynch = executionModeSynch;
        final ODistributedRequestId requestId = new ODistributedRequestId(dManager.getLocalNodeId(), dManager.getNextMessageIdCounter());
        final ODistributedTxContext ctx = localDistributedDatabase.registerTxContext(requestId);
        final AtomicBoolean lockReleased = new AtomicBoolean(true);
        try {
            acquireMultipleRecordLocks(iTx, maxAutoRetry, autoRetryDelay, eventListener, ctx);
            lockReleased.set(false);
            final List<ORecordOperation> uResult = (List<ORecordOperation>) OScenarioThreadLocal.executeAsDistributed(new Callable() {

                @Override
                public Object call() throws Exception {
                    return storage.commit(iTx, callback);
                }
            });
            try {
                localDistributedDatabase.getSyncConfiguration().setLastLSN(localNodeName, ((OLocalPaginatedStorage) storage.getUnderlying()).getLSN(), true);
            } catch (IOException e) {
                ODistributedServerLog.debug(this, dManager != null ? dManager.getLocalNodeName() : "?", null, ODistributedServerLog.DIRECTION.NONE, "Error on updating local LSN configuration for database '%s'", storage.getName());
            }
            // REMOVE THE TX OBJECT FROM DATABASE TO AVOID UND OPERATIONS ARE "LOST IN TRANSACTION"
            database.setDefaultTransactionMode();
            // After commit force the clean of dirty managers due to possible copy and miss clean.
            for (ORecordOperation ent : iTx.getAllRecordEntries()) {
                ORecordInternal.getDirtyManager(ent.getRecord()).clear();
            }
            final Set<String> involvedClusters = getInvolvedClusters(uResult);
            Set<String> nodes = getAvailableNodesButLocal(dbCfg, involvedClusters, localNodeName);
            if (nodes.isEmpty()) {
                // NO FURTHER NODES TO INVOLVE
                executionModeSynch = true;
                return null;
            }
            updateUndoTaskWithCreatedRecords(uResult, undoTasks);
            final OTxTaskResult localResult = createLocalTxResult(uResult);
            final OTxTask txTask = createTxTask(uResult);
            txTask.setLocalUndoTasks(undoTasks);
            try {
                txTask.setLastLSN(((OAbstractPaginatedStorage) storage.getUnderlying()).getLSN());
                OTransactionInternal.setStatus((OTransactionAbstract) iTx, OTransaction.TXSTATUS.COMMITTING);
                if (finalExecutionModeSynch) {
                    // SYNCHRONOUS, AUTO-RETRY IN CASE RECORDS ARE LOCKED
                    ODistributedResponse lastResult = null;
                    for (int retry = 1; retry <= maxAutoRetry; ++retry) {
                        boolean isLastRetry = maxAutoRetry == retry;
                        if (retry > 1) {
                            // REBUILD THE SERVER LIST
                            nodes = getAvailableNodesButLocal(dbCfg, involvedClusters, localNodeName);
                            if (nodes.isEmpty()) {
                                // NO FURTHER NODES TO INVOLVE
                                executionModeSynch = true;
                                return null;
                            }
                            ODistributedServerLog.debug(this, localNodeName, null, ODistributedServerLog.DIRECTION.NONE, "Retrying (%d/%d) transaction reqId=%s...", retry, maxAutoRetry, requestId);
                        }
                        // SYNCHRONOUS CALL: REPLICATE IT
                        lastResult = dManager.sendRequest(storage.getName(), involvedClusters, nodes, txTask, requestId.getMessageId(), EXECUTION_MODE.RESPONSE, localResult, null);
                        if (!processCommitResult(localNodeName, iTx, txTask, involvedClusters, uResult, nodes, autoRetryDelay, lastResult.getRequestId(), lastResult, isLastRetry)) {
                            // RETRY
                            Orient.instance().getProfiler().updateCounter("db." + database.getName() + ".distributedTxRetries", "Number of retries executed in distributed transaction", +1, "db.*.distributedTxRetries");
                            continue;
                        }
                        ODistributedServerLog.debug(this, localNodeName, null, ODistributedServerLog.DIRECTION.NONE, "Distributed transaction succeeded. Tasks: %s", txTask.getTasks());
                        // OK, DISTRIBUTED COMMIT SUCCEED
                        return null;
                    }
                    // ONLY CASE: ODistributedRecordLockedException MORE THAN AUTO-RETRY
                    ODistributedServerLog.debug(this, localNodeName, null, ODistributedServerLog.DIRECTION.NONE, "Distributed transaction retries exceed maximum auto-retries (%d). Task: %s - Tasks: %s", maxAutoRetry, txTask, txTask.getTasks());
                    // ROLLBACK TX
                    storage.executeUndoOnLocalServer(requestId, txTask);
                    sendTxCompleted(localNodeName, involvedClusters, nodes, lastResult.getRequestId(), false, txTask.getPartitionKey());
                    throw (RuntimeException) lastResult.getPayload();
                } else {
                    // ASYNC, MANAGE REPLICATION CALLBACK
                    final OCallable<Void, ODistributedRequestId> unlockCallback = new OCallable<Void, ODistributedRequestId>() {

                        @Override
                        public Void call(final ODistributedRequestId reqId) {
                            // FREE THE CONTEXT
                            if (lockReleased.compareAndSet(false, true)) {
                                localDistributedDatabase.popTxContext(requestId);
                                ctx.destroy();
                            }
                            return null;
                        }
                    };
                    executeAsyncTx(nodes, localResult, involvedClusters, txTask, requestId.getMessageId(), localNodeName, unlockCallback);
                }
            } catch (Throwable e) {
                // UNDO LOCAL TX
                storage.executeUndoOnLocalServer(requestId, txTask);
                executionModeSynch = true;
                if (e instanceof RuntimeException)
                    throw (RuntimeException) e;
                else if (e instanceof InterruptedException)
                    throw OException.wrapException(new ODistributedOperationException("Cannot commit transaction"), e);
                else
                    throw OException.wrapException(new ODistributedException("Cannot commit transaction"), e);
            }
        } catch (RuntimeException e) {
            executionModeSynch = true;
            throw e;
        } catch (InterruptedException e) {
            executionModeSynch = true;
            throw OException.wrapException(new ODistributedOperationException("Cannot commit transaction"), e);
        } catch (Exception e) {
            executionModeSynch = true;
            throw OException.wrapException(new ODistributedException("Cannot commit transaction"), e);
        } finally {
            if (executionModeSynch) {
                if (lockReleased.compareAndSet(false, true)) {
                    localDistributedDatabase.popTxContext(requestId);
                    ctx.destroy();
                }
            }
        }
    } catch (OValidationException e) {
        throw e;
    } catch (ODistributedRecordLockedException e) {
        throw e;
    } catch (OConcurrentCreateException e) {
        // REQUEST A REPAIR OF THE CLUSTER BECAUSE IS NOT ALIGNED
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairCluster(e.getActualRid().getClusterId());
        throw e;
    } catch (OConcurrentModificationException e) {
        localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord((ORecordId) e.getRid());
        throw e;
    } catch (Exception e) {
        for (ORecordOperation op : iTx.getAllRecordEntries()) {
            if (iTx.hasRecordCreation()) {
                final ORecordId lockEntireCluster = (ORecordId) op.getRID().copy();
                localDistributedDatabase.getDatabaseRepairer().enqueueRepairCluster(lockEntireCluster.getClusterId());
            }
            localDistributedDatabase.getDatabaseRepairer().enqueueRepairRecord((ORecordId) op.getRID());
        }
        storage.handleDistributedException("Cannot route TX operation against distributed node", e);
    }
    return null;
}

Example 69 with ORecordId

use of com.orientechnologies.orient.core.id.ORecordId in project orientdb by orientechnologies.

the class OIncrementalServerSync method importDelta.

/**
   * Deleted records are written in output stream first, then created/updated records. All records are sorted by record id.
   * <p>
   * Each record in output stream is written using following format:
   * <ol>
   * <li>Record's cluster id - 4 bytes</li>
   * <li>Record's cluster position - 8 bytes</li>
   * <li>Delete flag, 1 if record is deleted - 1 byte</li>
   * <li>Record version , only if record is not deleted - 4 bytes</li>
   * <li>Record type, only if record is not deleted - 1 byte</li>
   * <li>Length of binary presentation of record, only if record is not deleted - 4 bytes</li>
   * <li>Binary presentation of the record, only if record is not deleted - length of content is provided in above entity</li>
   * </ol>
   */
public void importDelta(final OServer serverInstance, final ODatabaseDocumentInternal db, final FileInputStream in, final String iNode) throws IOException {
    final String nodeName = serverInstance.getDistributedManager().getLocalNodeName();
    try {
        serverInstance.openDatabase(db);
        OScenarioThreadLocal.executeAsDistributed(new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                db.activateOnCurrentThread();
                long totalRecords = 0;
                long totalCreated = 0;
                long totalUpdated = 0;
                long totalDeleted = 0;
                long totalHoles = 0;
                long totalSkipped = 0;
                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "Started import of delta for database '" + db.getName() + "'");
                long lastLap = System.currentTimeMillis();
                // final GZIPInputStream gzipInput = new GZIPInputStream(in);
                try {
                    final DataInputStream input = new DataInputStream(in);
                    try {
                        final long records = input.readLong();
                        for (long i = 0; i < records; ++i) {
                            final int clusterId = input.readInt();
                            final long clusterPos = input.readLong();
                            final boolean deleted = input.readBoolean();
                            final ORecordId rid = new ORecordId(clusterId, clusterPos);
                            totalRecords++;
                            final OPaginatedCluster cluster = (OPaginatedCluster) db.getStorage().getUnderlying().getClusterById(rid.getClusterId());
                            final OPaginatedCluster.RECORD_STATUS recordStatus = cluster.getRecordStatus(rid.getClusterPosition());
                            ORecord newRecord = null;
                            if (deleted) {
                                ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- deleting %s", rid);
                                switch(recordStatus) {
                                    case REMOVED:
                                        // SKIP IT
                                        totalSkipped++;
                                        continue;
                                    case ALLOCATED:
                                    case PRESENT:
                                        // DELETE IT
                                        db.delete(rid);
                                        break;
                                    case NOT_EXISTENT:
                                        totalSkipped++;
                                        break;
                                }
                                totalDeleted++;
                            } else {
                                final int recordVersion = input.readInt();
                                final int recordType = input.readByte();
                                final int recordSize = input.readInt();
                                final byte[] recordContent = new byte[recordSize];
                                input.read(recordContent);
                                switch(recordStatus) {
                                    case REMOVED:
                                        // SKIP IT
                                        totalSkipped++;
                                        continue;
                                    case ALLOCATED:
                                    case PRESENT:
                                        // UPDATE IT
                                        newRecord = Orient.instance().getRecordFactoryManager().newInstance((byte) recordType);
                                        ORecordInternal.fill(newRecord, rid, ORecordVersionHelper.setRollbackMode(recordVersion), recordContent, true);
                                        final ORecord loadedRecord = rid.getRecord();
                                        if (loadedRecord instanceof ODocument) {
                                            // APPLY CHANGES FIELD BY FIELD TO MARK DIRTY FIELDS FOR INDEXES/HOOKS
                                            ODocument loadedDocument = (ODocument) loadedRecord;
                                            loadedDocument.merge((ODocument) newRecord, false, false);
                                            ORecordInternal.setVersion(loadedRecord, ORecordVersionHelper.setRollbackMode(recordVersion));
                                            loadedDocument.setDirty();
                                            newRecord = loadedDocument;
                                        }
                                        // SAVE THE UPDATE RECORD
                                        newRecord.save();
                                        ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- updating rid=%s type=%d size=%d v=%d content=%s", rid, recordType, recordSize, recordVersion, newRecord);
                                        totalUpdated++;
                                        break;
                                    case NOT_EXISTENT:
                                        // CREATE AND DELETE RECORD IF NEEDED
                                        do {
                                            newRecord = Orient.instance().getRecordFactoryManager().newInstance((byte) recordType);
                                            ORecordInternal.fill(newRecord, new ORecordId(rid.getClusterId(), -1), recordVersion - 1, recordContent, true);
                                            try {
                                                newRecord.save();
                                            } catch (ORecordNotFoundException e) {
                                                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "DELTA <- error on saving record (not found) rid=%s type=%d size=%d v=%d content=%s", rid, recordType, recordSize, recordVersion, newRecord);
                                            } catch (ORecordDuplicatedException e) {
                                                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "DELTA <- error on saving record (duplicated %s) rid=%s type=%d size=%d v=%d content=%s", e.getRid(), rid, recordType, recordSize, recordVersion, newRecord);
                                                // throw OException.wrapException(
                                                // new ODistributedDatabaseDeltaSyncException("Error on delta sync: found duplicated record " + rid), e);
                                                final ORecord duplicatedRecord = db.load(e.getRid(), null, true);
                                                if (duplicatedRecord == null) {
                                                    // RECORD REMOVED: THE INDEX IS DIRTY, FIX THE DIRTY INDEX
                                                    final ODocument doc = (ODocument) newRecord;
                                                    final OIndex<?> index = db.getMetadata().getIndexManager().getIndex(e.getIndexName());
                                                    final List<String> fields = index.getDefinition().getFields();
                                                    final List<Object> values = new ArrayList<Object>(fields.size());
                                                    for (String f : fields) {
                                                        values.add(doc.field(f));
                                                    }
                                                    final Object keyValue = index.getDefinition().createValue(values);
                                                    index.remove(keyValue, e.getRid());
                                                    // RESAVE THE RECORD
                                                    newRecord.save();
                                                } else
                                                    break;
                                            }
                                            if (newRecord.getIdentity().getClusterPosition() < clusterPos) {
                                                // DELETE THE RECORD TO CREATE A HOLE
                                                ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- creating hole rid=%s", newRecord.getIdentity());
                                                newRecord.delete();
                                                totalHoles++;
                                            }
                                        } while (newRecord.getIdentity().getClusterPosition() < clusterPos);
                                        ODistributedServerLog.debug(this, nodeName, iNode, DIRECTION.IN, "DELTA <- creating rid=%s type=%d size=%d v=%d content=%s", rid, recordType, recordSize, recordVersion, newRecord);
                                        totalCreated++;
                                        break;
                                }
                                if (newRecord.getIdentity().isPersistent() && !newRecord.getIdentity().equals(rid))
                                    throw new ODistributedDatabaseDeltaSyncException("Error on synchronization of records, rids are different: saved " + newRecord.getIdentity() + ", but it should be " + rid);
                            }
                            final long now = System.currentTimeMillis();
                            if (now - lastLap > 2000) {
                                // DUMP STATS EVERY SECOND
                                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "- %,d total entries: %,d created, %,d updated, %,d deleted, %,d holes, %,d skipped...", totalRecords, totalCreated, totalUpdated, totalDeleted, totalHoles, totalSkipped);
                                lastLap = now;
                            }
                        }
                        db.getMetadata().reload();
                    } finally {
                        input.close();
                    }
                } catch (Exception e) {
                    ODistributedServerLog.error(this, nodeName, iNode, DIRECTION.IN, "Error on installing database delta '%s' on local server", e, db.getName());
                    throw OException.wrapException(new ODistributedException("Error on installing database delta '" + db.getName() + "' on local server"), e);
                } finally {
                // gzipInput.close();
                }
                ODistributedServerLog.info(this, nodeName, iNode, DIRECTION.IN, "Installed database delta for '%s'. %d total entries: %d created, %d updated, %d deleted, %d holes, %,d skipped", db.getName(), totalRecords, totalCreated, totalUpdated, totalDeleted, totalHoles, totalSkipped);
                return null;
            }
        });
        db.activateOnCurrentThread();
    } catch (Exception e) {
        // FORCE FULL DATABASE SYNC
        ODistributedServerLog.error(this, nodeName, iNode, DIRECTION.IN, "Error while applying changes of database delta sync on '%s': forcing full database sync...", e, db.getName());
        throw OException.wrapException(new ODistributedDatabaseDeltaSyncException("Error while applying changes of database delta sync on '" + db.getName() + "': forcing full database sync..."), e);
    }
}

Example 70 with ORecordId

use of com.orientechnologies.orient.core.id.ORecordId in project orientdb by orientechnologies.

the class ODistributedDatabaseImpl method lockRecord.

@Override
public boolean lockRecord(final ORID rid, final ODistributedRequestId iRequestId, final long timeout) {
    // TODO: IMPROVE THIS BY RECEIVING THE RECORD AS PARAMETER INSTEAD OF RELOADING IT
    ORawBuffer originalRecord = null;
    if (rid.isPersistent()) {
        final ODatabaseDocumentInternal db = ODatabaseRecordThreadLocal.INSTANCE.getIfDefined();
        if (db != null)
            originalRecord = db.getStorage().getUnderlying().readRecord((ORecordId) rid, null, false, true, null).getResult();
    }
    final ODistributedLock lock = new ODistributedLock(iRequestId, originalRecord);
    boolean newLock = true;
    ODistributedLock currentLock = lockManager.putIfAbsent(rid, lock);
    if (currentLock != null) {
        if (iRequestId.equals(currentLock.reqId)) {
            // SAME ID, ALREADY LOCKED
            ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE, "Distributed transaction: %s locked record %s in database '%s' owned by %s (thread=%d)", iRequestId, rid, databaseName, currentLock.reqId, Thread.currentThread().getId());
            currentLock = null;
            newLock = false;
        } else {
            // TRY TO RE-LOCK IT UNTIL TIMEOUT IS EXPIRED
            final long startTime = System.currentTimeMillis();
            do {
                try {
                    if (timeout > 0) {
                        if (!currentLock.lock.await(timeout, TimeUnit.MILLISECONDS))
                            continue;
                    } else
                        currentLock.lock.await();
                    currentLock = lockManager.putIfAbsent(rid, lock);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    break;
                }
            } while (currentLock != null && (timeout == 0 || System.currentTimeMillis() - startTime < timeout));
        }
    }
    //
    if (ODistributedServerLog.isDebugEnabled())
        if (currentLock == null) {
            ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE, "Distributed transaction: %s locked record %s in database '%s' (thread=%d)", iRequestId, rid, databaseName, Thread.currentThread().getId());
        } else {
            ODistributedServerLog.debug(this, localNodeName, null, DIRECTION.NONE, "Distributed transaction: %s cannot lock record %s in database '%s' owned by %s (thread=%d)", iRequestId, rid, databaseName, currentLock.reqId, Thread.currentThread().getId());
        }
    if (currentLock != null)
        throw new ODistributedRecordLockedException(manager.getLocalNodeName(), rid, currentLock.reqId, timeout);
    return newLock;
}