Examples with OPhysicalPosition com.orientechnologies.orient.core.storage.OPhysicalPosition used on opensource projects

Example 16 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class LocalPaginatedClusterTest method testUpdateManyBigRecords.

public void testUpdateManyBigRecords() throws IOException {
    final int records = 5000;
    long seed = System.currentTimeMillis();
    Random mersenneTwisterFast = new Random(seed);
    System.out.println("testUpdateManyBigRecords seed : " + seed);
    Map<Long, byte[]> positionRecordMap = new HashMap<Long, byte[]>();
    Set<Long> updatedPositions = new HashSet<Long>();
    int recordVersion = 0;
    recordVersion++;
    recordVersion++;
    for (int i = 0; i < records; i++) {
        int recordSize = mersenneTwisterFast.nextInt(2 * OClusterPage.MAX_RECORD_SIZE) + OClusterPage.MAX_RECORD_SIZE + 1;
        byte[] bigRecord = new byte[recordSize];
        mersenneTwisterFast.nextBytes(bigRecord);
        final OPhysicalPosition physicalPosition = paginatedCluster.createRecord(bigRecord, recordVersion, (byte) 2, null);
        positionRecordMap.put(physicalPosition.clusterPosition, bigRecord);
    }
    int newRecordVersion = 0;
    newRecordVersion = recordVersion;
    newRecordVersion++;
    for (long clusterPosition : positionRecordMap.keySet()) {
        if (mersenneTwisterFast.nextBoolean()) {
            int recordSize = mersenneTwisterFast.nextInt(2 * OClusterPage.MAX_RECORD_SIZE) + OClusterPage.MAX_RECORD_SIZE + 1;
            byte[] bigRecord = new byte[recordSize];
            mersenneTwisterFast.nextBytes(bigRecord);
            paginatedCluster.updateRecord(clusterPosition, bigRecord, newRecordVersion, (byte) 3);
            positionRecordMap.put(clusterPosition, bigRecord);
            updatedPositions.add(clusterPosition);
        }
    }
    for (Map.Entry<Long, byte[]> entry : positionRecordMap.entrySet()) {
        ORawBuffer rawBuffer = paginatedCluster.readRecord(entry.getKey(), false);
        Assert.assertNotNull(rawBuffer);
        Assert.assertEquals(rawBuffer.buffer, entry.getValue());
        if (updatedPositions.contains(entry.getKey())) {
            Assert.assertEquals(rawBuffer.version, newRecordVersion);
            Assert.assertEquals(rawBuffer.recordType, 3);
        } else {
            Assert.assertEquals(rawBuffer.version, recordVersion);
            Assert.assertEquals(rawBuffer.recordType, 2);
        }
    }
}

Example 17 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class ODatabaseCompare method compareRecords.

@SuppressFBWarnings("NP_NULL_ON_SOME_PATH")
private boolean compareRecords(ODocumentHelper.RIDMapper ridMapper) {
    listener.onMessage("\nStarting deep comparison record by record. This may take a few minutes. Wait please...");
    Collection<String> clusterNames1 = makeDbCall(databaseOne, new ODbRelatedCall<Collection<String>>() {

        @Override
        public Collection<String> call(ODatabaseDocumentInternal database) {
            return database.getClusterNames();
        }
    });
    for (final String clusterName : clusterNames1) {
        // CHECK IF THE CLUSTER IS INCLUDED
        if (includeClusters != null) {
            if (!includeClusters.contains(clusterName))
                continue;
        } else if (excludeClusters != null) {
            if (excludeClusters.contains(clusterName))
                continue;
        }
        final int clusterId1 = makeDbCall(databaseOne, new ODbRelatedCall<Integer>() {

            @Override
            public Integer call(ODatabaseDocumentInternal database) {
                return database.getClusterIdByName(clusterName);
            }
        });
        final long[] db1Range = makeDbCall(databaseOne, new ODbRelatedCall<long[]>() {

            @Override
            public long[] call(ODatabaseDocumentInternal database) {
                return database.getStorage().getClusterDataRange(clusterId1);
            }
        });
        final long[] db2Range = makeDbCall(databaseTwo, new ODbRelatedCall<long[]>() {

            @Override
            public long[] call(ODatabaseDocumentInternal database) {
                return database.getStorage().getClusterDataRange(clusterId1);
            }
        });
        final long db1Max = db1Range[1];
        final long db2Max = db2Range[1];
        databaseOne.activateOnCurrentThread();
        final ODocument doc1 = new ODocument();
        databaseTwo.activateOnCurrentThread();
        final ODocument doc2 = new ODocument();
        final ORecordId rid = new ORecordId(clusterId1);
        // TODO why this maximums can be different?
        final long clusterMax = Math.max(db1Max, db2Max);
        final OStorage storage;
        ODatabaseDocumentInternal selectedDatabase;
        if (clusterMax == db1Max)
            selectedDatabase = databaseOne;
        else
            selectedDatabase = databaseTwo;
        OPhysicalPosition[] physicalPositions = makeDbCall(selectedDatabase, new ODbRelatedCall<OPhysicalPosition[]>() {

            @Override
            public OPhysicalPosition[] call(ODatabaseDocumentInternal database) {
                return database.getStorage().ceilingPhysicalPositions(clusterId1, new OPhysicalPosition(0));
            }
        });
        OStorageConfiguration configuration1 = makeDbCall(databaseOne, new ODbRelatedCall<OStorageConfiguration>() {

            @Override
            public OStorageConfiguration call(ODatabaseDocumentInternal database) {
                return database.getStorage().getConfiguration();
            }
        });
        OStorageConfiguration configuration2 = makeDbCall(databaseTwo, new ODbRelatedCall<OStorageConfiguration>() {

            @Override
            public OStorageConfiguration call(ODatabaseDocumentInternal database) {
                return database.getStorage().getConfiguration();
            }
        });
        String storageType1 = makeDbCall(databaseOne, new ODbRelatedCall<String>() {

            @Override
            public String call(ODatabaseDocumentInternal database) {
                return database.getStorage().getType();
            }
        });
        String storageType2 = makeDbCall(databaseTwo, new ODbRelatedCall<String>() {

            @Override
            public String call(ODatabaseDocumentInternal database) {
                return database.getStorage().getType();
            }
        });
        long recordsCounter = 0;
        while (physicalPositions.length > 0) {
            for (OPhysicalPosition physicalPosition : physicalPositions) {
                try {
                    recordsCounter++;
                    final long position = physicalPosition.clusterPosition;
                    rid.setClusterPosition(position);
                    if (rid.equals(new ORecordId(configuration1.indexMgrRecordId)) && rid.equals(new ORecordId(configuration2.indexMgrRecordId)))
                        continue;
                    if (rid.equals(new ORecordId(configuration1.schemaRecordId)) && rid.equals(new ORecordId(configuration2.schemaRecordId)))
                        continue;
                    if (rid.getClusterId() == 0 && rid.getClusterPosition() == 0) {
                        // Skip the compare of raw structure if the storage type are different, due the fact that are different by definition.
                        if (!storageType1.equals(storageType2))
                            continue;
                    }
                    final ORecordId rid2;
                    if (ridMapper == null)
                        rid2 = rid;
                    else {
                        final ORID newRid = ridMapper.map(rid);
                        if (newRid == null)
                            rid2 = rid;
                        else
                            rid2 = new ORecordId(newRid);
                    }
                    final ORawBuffer buffer1 = makeDbCall(databaseOne, new ODbRelatedCall<ORawBuffer>() {

                        @Override
                        public ORawBuffer call(ODatabaseDocumentInternal database) {
                            return database.getStorage().readRecord(rid, null, true, false, null).getResult();
                        }
                    });
                    final ORawBuffer buffer2 = makeDbCall(databaseTwo, new ODbRelatedCall<ORawBuffer>() {

                        @Override
                        public ORawBuffer call(ODatabaseDocumentInternal database) {
                            return database.getStorage().readRecord(rid2, null, true, false, null).getResult();
                        }
                    });
                    if (buffer1 == null && buffer2 == null)
                        // BOTH RECORD NULL, OK
                        continue;
                    else if (buffer1 == null && buffer2 != null) {
                        // REC1 NULL
                        listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " is null in DB1");
                        ++differences;
                    } else if (buffer1 != null && buffer2 == null) {
                        // REC2 NULL
                        listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " is null in DB2");
                        ++differences;
                    } else {
                        if (buffer1.recordType != buffer2.recordType) {
                            listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " recordType is different: " + (char) buffer1.recordType + " <-> " + (char) buffer2.recordType);
                            ++differences;
                        }
                        if (buffer1.buffer == null && buffer2.buffer == null) {
                        } else if (buffer1.buffer == null && buffer2.buffer != null) {
                            listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " content is different: null <-> " + buffer2.buffer.length);
                            ++differences;
                        } else if (buffer1.buffer != null && buffer2.buffer == null) {
                            listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " content is different: " + buffer1.buffer.length + " <-> null");
                            ++differences;
                        } else {
                            if (buffer1.recordType == ODocument.RECORD_TYPE) {
                                // DOCUMENT: TRY TO INSTANTIATE AND COMPARE
                                makeDbCall(databaseOne, new ODocumentHelper.ODbRelatedCall<Object>() {

                                    public Object call(ODatabaseDocumentInternal database) {
                                        doc1.reset();
                                        doc1.fromStream(buffer1.buffer);
                                        return null;
                                    }
                                });
                                makeDbCall(databaseTwo, new ODocumentHelper.ODbRelatedCall<Object>() {

                                    public Object call(ODatabaseDocumentInternal database) {
                                        doc2.reset();
                                        doc2.fromStream(buffer2.buffer);
                                        return null;
                                    }
                                });
                                if (rid.toString().equals(configuration1.schemaRecordId) && rid.toString().equals(configuration2.schemaRecordId)) {
                                    makeDbCall(databaseOne, new ODocumentHelper.ODbRelatedCall<java.lang.Object>() {

                                        public Object call(ODatabaseDocumentInternal database) {
                                            convertSchemaDoc(doc1);
                                            return null;
                                        }
                                    });
                                    makeDbCall(databaseTwo, new ODocumentHelper.ODbRelatedCall<java.lang.Object>() {

                                        public Object call(ODatabaseDocumentInternal database) {
                                            convertSchemaDoc(doc2);
                                            return null;
                                        }
                                    });
                                }
                                if (!ODocumentHelper.hasSameContentOf(doc1, databaseOne, doc2, databaseTwo, ridMapper)) {
                                    listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " document content is different");
                                    listener.onMessage("\n--- REC1: " + new String(buffer1.buffer));
                                    listener.onMessage("\n--- REC2: " + new String(buffer2.buffer));
                                    listener.onMessage("\n");
                                    ++differences;
                                }
                            } else {
                                if (buffer1.buffer.length != buffer2.buffer.length) {
                                    // CHECK IF THE TRIMMED SIZE IS THE SAME
                                    final String rec1 = new String(buffer1.buffer).trim();
                                    final String rec2 = new String(buffer2.buffer).trim();
                                    if (rec1.length() != rec2.length()) {
                                        listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " content length is different: " + buffer1.buffer.length + " <-> " + buffer2.buffer.length);
                                        if (buffer1.recordType == ODocument.RECORD_TYPE)
                                            listener.onMessage("\n--- REC1: " + rec1);
                                        if (buffer2.recordType == ODocument.RECORD_TYPE)
                                            listener.onMessage("\n--- REC2: " + rec2);
                                        listener.onMessage("\n");
                                        ++differences;
                                    }
                                } else {
                                    // CHECK BYTE PER BYTE
                                    for (int b = 0; b < buffer1.buffer.length; ++b) {
                                        if (buffer1.buffer[b] != buffer2.buffer[b]) {
                                            listener.onMessage("\n- ERR: RID=" + clusterId1 + ":" + position + " content is different at byte #" + b + ": " + buffer1.buffer[b] + " <-> " + buffer2.buffer[b]);
                                            listener.onMessage("\n--- REC1: " + new String(buffer1.buffer));
                                            listener.onMessage("\n--- REC2: " + new String(buffer2.buffer));
                                            listener.onMessage("\n");
                                            ++differences;
                                            break;
                                        }
                                    }
                                }
                            }
                        }
                    }
                } catch (RuntimeException e) {
                    OLogManager.instance().error(this, "Error during data comparison of records with rid " + rid);
                    throw e;
                }
            }
            final OPhysicalPosition[] curPosition = physicalPositions;
            physicalPositions = makeDbCall(selectedDatabase, new ODbRelatedCall<OPhysicalPosition[]>() {

                @Override
                public OPhysicalPosition[] call(ODatabaseDocumentInternal database) {
                    return database.getStorage().higherPhysicalPositions(clusterId1, curPosition[curPosition.length - 1]);
                }
            });
            if (recordsCounter % 10000 == 0)
                listener.onMessage("\n" + recordsCounter + " records were processed for cluster " + clusterName + " ...");
        }
        listener.onMessage("\nCluster comparison was finished, " + recordsCounter + " records were processed for cluster " + clusterName + " ...");
    }
    return true;
}

Example 18 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class ODatabaseImport method migrateLinksInImportedDocuments.

private void migrateLinksInImportedDocuments(Set<ORID> brokenRids) throws IOException {
    listener.onMessage("\n\nStarted migration of links (-migrateLinks=true). Links are going to be updated according to new RIDs:");
    final long begin = System.currentTimeMillis();
    long last = begin;
    long documentsLastLap = 0;
    long totalDocuments = 0;
    Collection<String> clusterNames = database.getClusterNames();
    for (String clusterName : clusterNames) {
        if (OMetadataDefault.CLUSTER_INDEX_NAME.equals(clusterName) || OMetadataDefault.CLUSTER_INTERNAL_NAME.equals(clusterName) || OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME.equals(clusterName))
            continue;
        long documents = 0;
        String prefix = "";
        listener.onMessage("\n- Cluster " + clusterName + "...");
        final int clusterId = database.getClusterIdByName(clusterName);
        final long clusterRecords = database.countClusterElements(clusterId);
        OStorage storage = database.getStorage();
        OPhysicalPosition[] positions = storage.ceilingPhysicalPositions(clusterId, new OPhysicalPosition(0));
        while (positions.length > 0) {
            for (OPhysicalPosition position : positions) {
                ORecord record = database.load(new ORecordId(clusterId, position.clusterPosition));
                if (record instanceof ODocument) {
                    ODocument document = (ODocument) record;
                    rewriteLinksInDocument(document, brokenRids);
                    documents++;
                    documentsLastLap++;
                    totalDocuments++;
                    final long now = System.currentTimeMillis();
                    if (now - last > IMPORT_RECORD_DUMP_LAP_EVERY_MS) {
                        listener.onMessage(String.format("\n--- Migrated %,d of %,d records (%,.2f/sec)", documents, clusterRecords, (float) documentsLastLap * 1000 / (float) IMPORT_RECORD_DUMP_LAP_EVERY_MS));
                        // RESET LAP COUNTERS
                        last = now;
                        documentsLastLap = 0;
                        prefix = "\n---";
                    }
                }
            }
            positions = storage.higherPhysicalPositions(clusterId, positions[positions.length - 1]);
        }
        listener.onMessage(String.format("%s Completed migration of %,d records in current cluster", prefix, documents));
    }
    listener.onMessage(String.format("\nTotal links updated: %,d", totalDocuments));
}

Example 19 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class ODatabaseImport method migrateLinksInImportedDocuments.

private void migrateLinksInImportedDocuments() throws IOException {
    listener.onMessage("\n\nStarted migration of links (-migrateLinks=true). Links are going to be updated according to new RIDs:");
    final long begin = System.currentTimeMillis();
    long last = begin;
    long documentsLastLap = 0;
    long totalDocuments = 0;
    Collection<String> clusterNames = database.getClusterNames();
    for (String clusterName : clusterNames) {
        if (OMetadataDefault.CLUSTER_INDEX_NAME.equals(clusterName) || OMetadataDefault.CLUSTER_INTERNAL_NAME.equals(clusterName) || OMetadataDefault.CLUSTER_MANUAL_INDEX_NAME.equals(clusterName))
            continue;
        long documents = 0;
        String prefix = "";
        listener.onMessage("\n- Cluster " + clusterName + "...");
        final int clusterId = database.getClusterIdByName(clusterName);
        final long clusterRecords = database.countClusterElements(clusterId);
        OStorage storage = database.getStorage();
        OPhysicalPosition[] positions = storage.ceilingPhysicalPositions(clusterId, new OPhysicalPosition(0));
        while (positions.length > 0) {
            for (OPhysicalPosition position : positions) {
                ORecord record = database.load(new ORecordId(clusterId, position.clusterPosition));
                if (record instanceof ODocument) {
                    ODocument document = (ODocument) record;
                    rewriteLinksInDocument(document);
                    documents++;
                    documentsLastLap++;
                    totalDocuments++;
                    final long now = System.currentTimeMillis();
                    if (now - last > IMPORT_RECORD_DUMP_LAP_EVERY_MS) {
                        listener.onMessage(String.format("\n--- Migrated %,d of %,d records (%,.2f/sec)", documents, clusterRecords, (float) documentsLastLap * 1000 / (float) IMPORT_RECORD_DUMP_LAP_EVERY_MS));
                        // RESET LAP COUNTERS
                        last = now;
                        documentsLastLap = 0;
                        prefix = "\n---";
                    }
                }
            }
            positions = storage.higherPhysicalPositions(clusterId, positions[positions.length - 1]);
        }
        listener.onMessage(String.format("%s Completed migration of %,d records in current cluster", prefix, documents));
    }
    listener.onMessage(String.format("\nTotal links updated: %,d", totalDocuments));
}

Example 20 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class LocalPaginatedClusterTest method testAllocatePositionMap.

public void testAllocatePositionMap() throws IOException {
    OPhysicalPosition position = paginatedCluster.allocatePosition((byte) 'd');
    Assert.assertTrue(position.clusterPosition >= 0);
    ORawBuffer rec = paginatedCluster.readRecord(position.clusterPosition, false);
    Assert.assertNull(rec);
    paginatedCluster.createRecord(new byte[20], 1, (byte) 'd', position);
    rec = paginatedCluster.readRecord(position.clusterPosition, false);
    Assert.assertNotNull(rec);
}