Examples with OPhysicalPosition com.orientechnologies.orient.core.storage.OPhysicalPosition used on opensource projects

Example 36 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class LocalPaginatedClusterTest method testRemoveHalfRecords.

public void testRemoveHalfRecords() throws IOException {
    final int records = 10000;
    long seed = System.currentTimeMillis();
    Random mersenneTwisterFast = new Random(seed);
    System.out.println("testRemoveHalfRecords seed : " + seed);
    Map<Long, byte[]> positionRecordMap = new HashMap<Long, byte[]>();
    int recordVersion = 0;
    recordVersion++;
    recordVersion++;
    for (int i = 0; i < records; i++) {
        int recordSize = mersenneTwisterFast.nextInt(3 * OClusterPage.MAX_RECORD_SIZE) + 1;
        byte[] bigRecord = new byte[recordSize];
        mersenneTwisterFast.nextBytes(bigRecord);
        final OPhysicalPosition physicalPosition = paginatedCluster.createRecord(bigRecord, recordVersion, (byte) 2, null);
        positionRecordMap.put(physicalPosition.clusterPosition, bigRecord);
    }
    int deletedRecords = 0;
    Assert.assertEquals(records, paginatedCluster.getEntries());
    Set<Long> deletedPositions = new HashSet<Long>();
    Iterator<Long> positionIterator = positionRecordMap.keySet().iterator();
    while (positionIterator.hasNext()) {
        long clusterPosition = positionIterator.next();
        if (mersenneTwisterFast.nextBoolean()) {
            deletedPositions.add(clusterPosition);
            Assert.assertTrue(paginatedCluster.deleteRecord(clusterPosition));
            deletedRecords++;
            Assert.assertEquals(records - deletedRecords, paginatedCluster.getEntries());
            positionIterator.remove();
        }
    }
    Assert.assertEquals(paginatedCluster.getEntries(), records - deletedRecords);
    for (long deletedPosition : deletedPositions) {
        Assert.assertNull(paginatedCluster.readRecord(deletedPosition, false));
        Assert.assertFalse(paginatedCluster.deleteRecord(deletedPosition));
    }
    for (Map.Entry<Long, byte[]> entry : positionRecordMap.entrySet()) {
        ORawBuffer rawBuffer = paginatedCluster.readRecord(entry.getKey(), false);
        Assert.assertNotNull(rawBuffer);
        Assert.assertEquals(rawBuffer.version, recordVersion);
        Assert.assertEquals(rawBuffer.buffer, entry.getValue());
        Assert.assertEquals(rawBuffer.recordType, 2);
    }
}

Example 37 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class LocalPaginatedClusterTest method testHideHalfRecords.

public void testHideHalfRecords() throws IOException {
    final int records = 10000;
    long seed = System.currentTimeMillis();
    Random mersenneTwisterFast = new Random(seed);
    System.out.println("testHideHalfRecords seed : " + seed);
    Map<Long, byte[]> positionRecordMap = new HashMap<Long, byte[]>();
    int recordVersion = 0;
    recordVersion++;
    recordVersion++;
    for (int i = 0; i < records; i++) {
        int recordSize = mersenneTwisterFast.nextInt(3 * OClusterPage.MAX_RECORD_SIZE) + 1;
        byte[] bigRecord = new byte[recordSize];
        mersenneTwisterFast.nextBytes(bigRecord);
        final OPhysicalPosition physicalPosition = paginatedCluster.createRecord(bigRecord, recordVersion, (byte) 2, null);
        positionRecordMap.put(physicalPosition.clusterPosition, bigRecord);
    }
    int hiddenRecords = 0;
    Assert.assertEquals(records, paginatedCluster.getEntries());
    Set<Long> hiddenPositions = new HashSet<Long>();
    Iterator<Long> positionIterator = positionRecordMap.keySet().iterator();
    while (positionIterator.hasNext()) {
        long clusterPosition = positionIterator.next();
        if (mersenneTwisterFast.nextBoolean()) {
            hiddenPositions.add(clusterPosition);
            Assert.assertTrue(paginatedCluster.hideRecord(clusterPosition));
            hiddenRecords++;
            Assert.assertEquals(records - hiddenRecords, paginatedCluster.getEntries());
            positionIterator.remove();
        }
    }
    Assert.assertEquals(paginatedCluster.getEntries(), records - hiddenRecords);
    for (long deletedPosition : hiddenPositions) {
        Assert.assertNull(paginatedCluster.readRecord(deletedPosition, false));
        Assert.assertFalse(paginatedCluster.hideRecord(deletedPosition));
    }
    for (Map.Entry<Long, byte[]> entry : positionRecordMap.entrySet()) {
        ORawBuffer rawBuffer = paginatedCluster.readRecord(entry.getKey(), false);
        Assert.assertNotNull(rawBuffer);
        Assert.assertEquals(rawBuffer.version, recordVersion);
        Assert.assertEquals(rawBuffer.buffer, entry.getValue());
        Assert.assertEquals(rawBuffer.recordType, 2);
    }
}

Example 38 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class LocalPaginatedStorageCreateCrashRestoreIT method compareDocuments.

private void compareDocuments(long lastTs) {
    long minTs = Long.MAX_VALUE;
    baseDocumentTx.activateOnCurrentThread();
    int clusterId = baseDocumentTx.getClusterIdByName("TestClass");
    OStorage baseStorage = baseDocumentTx.getStorage();
    OPhysicalPosition[] physicalPositions = baseStorage.ceilingPhysicalPositions(clusterId, new OPhysicalPosition(0));
    int recordsRestored = 0;
    int recordsTested = 0;
    while (physicalPositions.length > 0) {
        final ORecordId rid = new ORecordId(clusterId);
        for (OPhysicalPosition physicalPosition : physicalPositions) {
            rid.setClusterPosition(physicalPosition.clusterPosition);
            baseDocumentTx.activateOnCurrentThread();
            ODocument baseDocument = baseDocumentTx.load(rid);
            testDocumentTx.activateOnCurrentThread();
            List<ODocument> testDocuments = testDocumentTx.query(new OSQLSynchQuery<ODocument>("select from TestClass where id  = " + baseDocument.field("id")));
            if (testDocuments.size() == 0) {
                if (((Long) baseDocument.field("timestamp")) < minTs)
                    minTs = baseDocument.field("timestamp");
            } else {
                ODocument testDocument = testDocuments.get(0);
                Assert.assertEquals((Object) testDocument.field("id"), baseDocument.field("id"));
                Assert.assertEquals((Object) testDocument.field("timestamp"), baseDocument.field("timestamp"));
                Assert.assertEquals((Object) testDocument.field("stringValue"), baseDocument.field("stringValue"));
                recordsRestored++;
            }
            recordsTested++;
            if (recordsTested % 10000 == 0)
                System.out.println(recordsTested + " were tested, " + recordsRestored + " were restored ...");
        }
        physicalPositions = baseStorage.higherPhysicalPositions(clusterId, physicalPositions[physicalPositions.length - 1]);
    }
    System.out.println(recordsRestored + " records were restored. Total records " + recordsTested + ". Max interval for lost records " + (lastTs - minTs));
}

Example 39 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class LocalPaginatedStorageSmallCacheBigRecordsCrashRestoreIT method compareDocuments.

private void compareDocuments(long lastTs) {
    long minTs = Long.MAX_VALUE;
    baseDocumentTx.activateOnCurrentThread();
    int clusterId = baseDocumentTx.getClusterIdByName("TestClass");
    OStorage baseStorage = baseDocumentTx.getStorage();
    OPhysicalPosition[] physicalPositions = baseStorage.ceilingPhysicalPositions(clusterId, new OPhysicalPosition(0));
    int recordsRestored = 0;
    int recordsTested = 0;
    while (physicalPositions.length > 0) {
        final ORecordId rid = new ORecordId(clusterId);
        for (OPhysicalPosition physicalPosition : physicalPositions) {
            rid.setClusterPosition(physicalPosition.clusterPosition);
            baseDocumentTx.activateOnCurrentThread();
            ODocument baseDocument = baseDocumentTx.load(rid);
            testDocumentTx.activateOnCurrentThread();
            List<ODocument> testDocuments = testDocumentTx.query(new OSQLSynchQuery<ODocument>("select from TestClass where id  = " + baseDocument.field("id")));
            if (testDocuments.size() == 0) {
                if (((Long) baseDocument.field("timestamp")) < minTs) {
                    minTs = baseDocument.field("timestamp");
                }
            } else {
                ODocument testDocument = testDocuments.get(0);
                assertThat(testDocument.field("id")).as("id:: %s", testDocument.field("id")).isEqualTo(baseDocument.field("id"));
                assertThat(testDocument.field("timestamp")).as("documents:: %s - %s", testDocument, baseDocument).isEqualTo(baseDocument.field("timestamp"));
                assertThat(testDocument.field("stringValue")).as("id:: %s", testDocument.field("id")).isEqualTo(baseDocument.field("stringValue"));
                assertThat(testDocument.field("binaryValue")).as("id:: %s", testDocument.field("id")).isEqualTo(baseDocument.field("binaryValue"));
                recordsRestored++;
            }
            recordsTested++;
            if (recordsTested % 10000 == 0)
                System.out.println(recordsTested + " were tested, " + recordsRestored + " were restored ...");
        }
        physicalPositions = baseStorage.higherPhysicalPositions(clusterId, physicalPositions[physicalPositions.length - 1]);
    }
    System.out.println(recordsRestored + " records were restored. Total records " + recordsTested + ". Max interval for lost records " + (lastTs - minTs));
}

Example 40 with OPhysicalPosition

use of com.orientechnologies.orient.core.storage.OPhysicalPosition in project orientdb by orientechnologies.

the class LocalPaginatedStorageMixCrashRestoreIT method compareDocuments.

private void compareDocuments(long lastTs) {
    long minTs = Long.MAX_VALUE;
    baseDocumentTx.activateOnCurrentThread();
    int clusterId = baseDocumentTx.getClusterIdByName("TestClass");
    OStorage baseStorage = baseDocumentTx.getStorage();
    OPhysicalPosition[] physicalPositions = baseStorage.ceilingPhysicalPositions(clusterId, new OPhysicalPosition(0));
    int recordsRestored = 0;
    int recordsTested = 0;
    while (physicalPositions.length > 0) {
        final ORecordId rid = new ORecordId(clusterId);
        for (OPhysicalPosition physicalPosition : physicalPositions) {
            rid.setClusterPosition(physicalPosition.clusterPosition);
            ODatabaseRecordThreadLocal.INSTANCE.set(baseDocumentTx);
            ODocument baseDocument = baseDocumentTx.load(rid);
            int id = baseDocument.<Integer>field("id");
            if (addedIds.contains(id)) {
                ODatabaseRecordThreadLocal.INSTANCE.set(testDocumentTx);
                List<ODocument> testDocuments = testDocumentTx.query(new OSQLSynchQuery<ODocument>("select from TestClass where id  = " + baseDocument.field("id")));
                if (testDocuments.size() == 0) {
                    if (((Long) baseDocument.field("timestamp")) < minTs)
                        minTs = baseDocument.field("timestamp");
                } else {
                    ODocument testDocument = testDocuments.get(0);
                    Assert.assertEquals(testDocument.field("id"), baseDocument.field("id"));
                    Assert.assertEquals(testDocument.field("timestamp"), baseDocument.field("timestamp"));
                    Assert.assertEquals(testDocument.field("stringValue"), baseDocument.field("stringValue"));
                    recordsRestored++;
                }
                recordsTested++;
            } else if (updatedIds.contains(id)) {
                ODatabaseRecordThreadLocal.INSTANCE.set(testDocumentTx);
                List<ODocument> testDocuments = testDocumentTx.query(new OSQLSynchQuery<ODocument>("select from TestClass where id  = " + baseDocument.field("id")));
                if (testDocuments.size() == 0) {
                    if (((Long) baseDocument.field("timestamp")) < minTs)
                        minTs = baseDocument.field("timestamp");
                } else {
                    ODocument testDocument = testDocuments.get(0);
                    if (testDocument.field("timestamp").equals(baseDocument.field("timestamp")) && testDocument.field("stringValue").equals(baseDocument.field("stringValue"))) {
                        recordsRestored++;
                    } else {
                        if (((Long) baseDocument.field("timestamp")) < minTs)
                            minTs = baseDocument.field("timestamp");
                    }
                }
                recordsTested++;
            }
            if (recordsTested % 10000 == 0)
                System.out.println(recordsTested + " were tested, " + recordsRestored + " were restored ...");
        }
        physicalPositions = baseStorage.higherPhysicalPositions(clusterId, physicalPositions[physicalPositions.length - 1]);
    }
    ODatabaseRecordThreadLocal.INSTANCE.set(testDocumentTx);
    System.out.println("Check deleted records");
    for (Map.Entry<Integer, Long> deletedEntry : deletedIds.entrySet()) {
        int deletedId = deletedEntry.getKey();
        List<ODocument> testDocuments = testDocumentTx.query(new OSQLSynchQuery<ODocument>("select from TestClass where id  = " + deletedId));
        if (!testDocuments.isEmpty()) {
            if (deletedEntry.getValue() < minTs)
                minTs = deletedEntry.getValue();
        } else
            recordsRestored++;
        recordsTested++;
    }
    System.out.println("Deleted records were checked." + deletedIds.size() + " were verified.");
    System.out.println(recordsRestored + " records were restored. Total records " + recordsTested + ". Max interval for lost records " + (lastTs - minTs));
}