Examples with LongArrayList jetbrains.exodus.core.dataStructures.LongArrayList used on opensource projects

Example 1 with LongArrayList

use of jetbrains.exodus.core.dataStructures.LongArrayList in project xodus by JetBrains.

the class GarbageCollector method deletePendingFiles.

void deletePendingFiles() {
    cleaner.checkThread();
    final LongArrayList filesToDelete = new LongArrayList();
    Long fileAddress;
    while ((fileAddress = deletionQueue.poll()) != null) {
        if (pendingFilesToDelete.remove(fileAddress)) {
            filesToDelete.add(fileAddress);
        }
    }
    if (!filesToDelete.isEmpty()) {
        // force flush and fsync in order to fix XD-249
        // in order to avoid data loss, it's necessary to make sure that any GC transaction is flushed
        // to underlying storage device before any file is deleted
        env.flushAndSync();
        RemoveBlockType rbt = ec.getGcRenameFiles() ? RemoveBlockType.Rename : RemoveBlockType.Delete;
        env.removeFiles(filesToDelete.toArray(), rbt);
    }
}

Example 2 with LongArrayList

use of jetbrains.exodus.core.dataStructures.LongArrayList in project xodus by JetBrains.

the class EntityIdArrayCachedInstanceIterableFactory method createInstance.

public static CachedInstanceIterable createInstance(@NotNull final PersistentStoreTransaction txn, @NotNull final EntityIterableBase source, @NotNull final EntityIteratorBase it) {
    try {
        if (!it.hasNext()) {
            return new EmptyCachedInstanceIterable(txn, source);
        } else {
            final IntArrayList typeIds = IntArrayListSpinAllocator.alloc();
            final LongArrayList localIds = LongArrayListSpinAllocator.alloc();
            long min;
            long max;
            try {
                boolean onlyOneTypeId = true;
                boolean localSorted = true;
                if (source.isSortedById()) {
                    int lastTypeId = -1;
                    EntityId id = it.nextId();
                    while (true) {
                        final int nextTypeId;
                        if (id == null) {
                            nextTypeId = NULL_TYPE_ID;
                            localIds.add(0);
                        } else {
                            nextTypeId = id.getTypeId();
                            localIds.add(id.getLocalId());
                        }
                        if (nextTypeId != lastTypeId) {
                            if (lastTypeId != -1) {
                                onlyOneTypeId = false;
                                // add upper boundary for previous
                                typeIds.add(localIds.size() - 1);
                            }
                            typeIds.add(nextTypeId);
                            lastTypeId = nextTypeId;
                        }
                        if (!it.hasNext()) {
                            if (!onlyOneTypeId) {
                                // add boundary for last
                                typeIds.add(localIds.size());
                            }
                            break;
                        }
                        id = it.nextId();
                    }
                    min = localIds.get(0);
                    max = localIds.get(localIds.size() - 1);
                } else {
                    int lastTypeId = -1;
                    long lastLocalId = -1;
                    min = Long.MAX_VALUE;
                    max = Long.MIN_VALUE;
                    boolean compact = true;
                    EntityId id = it.nextId();
                    while (true) {
                        final int nextTypeId;
                        final long nextLocalId;
                        if (id == null) {
                            nextTypeId = NULL_TYPE_ID;
                            nextLocalId = 0;
                        } else {
                            nextTypeId = id.getTypeId();
                            nextLocalId = id.getLocalId();
                        }
                        if (localSorted) {
                            if (lastTypeId > nextTypeId || lastTypeId == nextTypeId && lastLocalId > nextLocalId) {
                                final int length;
                                if (nextTypeId == NULL_TYPE_ID && (length = localIds.size()) <= 1) {
                                    if (length == 1) {
                                        // direct conversion
                                        onlyOneTypeId = false;
                                        localSorted = false;
                                        compact = false;
                                    } else {
                                        typeIds.add(NULL_TYPE_ID);
                                    }
                                    lastLocalId = nextLocalId;
                                } else {
                                    localSorted = false;
                                }
                            } else {
                                lastLocalId = nextLocalId;
                            }
                        }
                        localIds.add(nextLocalId);
                        if (nextLocalId > max) {
                            max = nextLocalId;
                        }
                        if (nextLocalId < min) {
                            min = nextLocalId;
                        }
                        if (compact) {
                            if (localSorted) {
                                if (nextTypeId > lastTypeId) {
                                    if (lastTypeId != -1) {
                                        onlyOneTypeId = false;
                                        // add upper boundary for previous
                                        typeIds.add(localIds.size() - 1);
                                    }
                                    typeIds.add(nextTypeId);
                                }
                                lastTypeId = nextTypeId;
                            } else {
                                if (typeIds.size() > 1 || nextTypeId != lastTypeId) {
                                    onlyOneTypeId = false;
                                    compact = false;
                                    addNextTypeId(nextTypeId, typeIds, localIds);
                                }
                            }
                        } else {
                            typeIds.add(nextTypeId);
                        }
                        if (!it.hasNext()) {
                            if (compact && !onlyOneTypeId) {
                                // add boundary for last
                                typeIds.add(localIds.size());
                            }
                            break;
                        }
                        id = it.nextId();
                    }
                }
                if (localSorted) {
                    if (onlyOneTypeId) {
                        return makeSingleTypeSortedIterable(txn, source, it, typeIds, localIds, min, max);
                    } else {
                        return new MultiTypeSortedEntityIdArrayCachedInstanceIterable(txn, source, typeIds.toArray(), localIds.toArray(), it.toSet());
                    }
                } else {
                    if (onlyOneTypeId) {
                        return makeSingleTypeUnsortedIterable(txn, source, it, typeIds, localIds, min, max);
                    } else {
                        return new MultiTypeUnsortedEntityIdArrayCachedInstanceIterable(txn, source, typeIds.toArray(), localIds.toArray(), it.toSet());
                    }
                }
            } finally {
                LongArrayListSpinAllocator.dispose(localIds);
                IntArrayListSpinAllocator.dispose(typeIds);
            }
        }
    } finally {
        it.disposeIfShouldBe();
    }
}

Example 3 with LongArrayList

use of jetbrains.exodus.core.dataStructures.LongArrayList in project xodus by JetBrains.

the class LogTests method testWriteSequentialRead.

private void testWriteSequentialRead(int fileSize, int pageSize) {
    initLog(fileSize, pageSize);
    final int count = 50000;
    final LongArrayList addrs = new LongArrayList();
    log.beginWrite();
    for (int i = 0; i < count; ++i) {
        addrs.add(writeData(CompressedUnsignedLongByteIterable.getIterable(i)));
    }
    log.endWrite();
    for (int i = 0; i < count; ++i) {
        Assert.assertEquals(i, (int) CompressedUnsignedLongByteIterable.getLong(getLog().read(addrs.get(i)).getData()));
    }
}

Example 4 with LongArrayList

use of jetbrains.exodus.core.dataStructures.LongArrayList in project xodus by JetBrains.

the class FileSystemBlobVaultOld method flushBlobs.

@Override
public void flushBlobs(@Nullable final LongHashMap<InputStream> blobStreams, @Nullable final LongHashMap<File> blobFiles, @Nullable final LongSet deferredBlobsToDelete, @NotNull final Transaction txn) throws Exception {
    if (blobStreams != null) {
        blobStreams.forEachEntry(new ObjectProcedureThrows<Map.Entry<Long, InputStream>, Exception>() {

            @Override
            public boolean execute(final Map.Entry<Long, InputStream> object) throws Exception {
                final InputStream stream = object.getValue();
                stream.reset();
                setContent(object.getKey(), stream);
                return true;
            }
        });
    }
    // if there were blob files then move them
    if (blobFiles != null) {
        blobFiles.forEachEntry(new ObjectProcedureThrows<Map.Entry<Long, File>, Exception>() {

            @Override
            public boolean execute(final Map.Entry<Long, File> object) throws Exception {
                setContent(object.getKey(), object.getValue());
                return true;
            }
        });
    }
    // if there are deferred blobs to delete then defer their deletion
    if (deferredBlobsToDelete != null) {
        final LongArrayList copy = new LongArrayList(deferredBlobsToDelete.size());
        final LongIterator it = deferredBlobsToDelete.iterator();
        while (it.hasNext()) {
            copy.add(it.nextLong());
        }
        final Environment environment = txn.getEnvironment();
        environment.executeTransactionSafeTask(new Runnable() {

            @Override
            public void run() {
                DeferredIO.getJobProcessor().queueIn(new Job() {

                    @Override
                    protected void execute() {
                        final long[] blobHandles = copy.getInstantArray();
                        for (int i = 0; i < copy.size(); ++i) {
                            delete(blobHandles[i]);
                        }
                    }

                    @Override
                    public String getName() {
                        return "Delete obsolete blob files";
                    }

                    @Override
                    public String getGroup() {
                        return environment.getLocation();
                    }
                }, environment.getEnvironmentConfig().getGcFilesDeletionDelay());
            }
        });
    }
}

Example 5 with LongArrayList

use of jetbrains.exodus.core.dataStructures.LongArrayList in project xodus by JetBrains.

the class Log method setHighAddress.

@SuppressWarnings({ "OverlyLongMethod" })
public LogTip setHighAddress(final LogTip logTip, final long highAddress) {
    if (highAddress > logTip.highAddress) {
        throw new ExodusException("Only can decrease high address");
    }
    if (highAddress == logTip.highAddress) {
        if (bufferedWriter != null) {
            throw new IllegalStateException("Unexpected write in progress");
        }
        return logTip;
    }
    final LogFileSet.Mutable fileSetMutable = logTip.logFileSet.beginWrite();
    // begin of test-only code
    final LogTestConfig testConfig = this.testConfig;
    if (testConfig != null && testConfig.isSettingHighAddressDenied()) {
        throw new ExodusException("Setting high address is denied");
    }
    // end of test-only code
    // at first, remove all files which are higher than highAddress
    closeWriter();
    final LongArrayList blocksToDelete = new LongArrayList();
    long blockToTruncate = -1L;
    for (final long blockAddress : fileSetMutable.getArray()) {
        if (blockAddress <= highAddress) {
            blockToTruncate = blockAddress;
            break;
        }
        blocksToDelete.add(blockAddress);
    }
    // truncate log
    for (int i = 0; i < blocksToDelete.size(); ++i) {
        removeFile(blocksToDelete.get(i), RemoveBlockType.Delete, fileSetMutable);
    }
    if (blockToTruncate >= 0) {
        truncateFile(blockToTruncate, highAddress - blockToTruncate);
    }
    final LogTip updatedTip;
    if (fileSetMutable.isEmpty()) {
        updateLogIdentity();
        updatedTip = new LogTip(fileSize);
    } else {
        final long oldHighPageAddress = logTip.pageAddress;
        long approvedHighAddress = logTip.approvedHighAddress;
        if (highAddress < approvedHighAddress) {
            approvedHighAddress = highAddress;
        }
        final long highPageAddress = getHighPageAddress(highAddress);
        final LogFileSet.Immutable fileSetImmutable = fileSetMutable.endWrite();
        final int highPageSize = (int) (highAddress - highPageAddress);
        if (oldHighPageAddress == highPageAddress) {
            updatedTip = logTip.withResize(highPageSize, highAddress, approvedHighAddress, fileSetImmutable);
        } else {
            updateLogIdentity();
            final byte[] highPageContent = new byte[cachePageSize];
            if (highPageSize > 0 && readBytes(highPageContent, highPageAddress) < highPageSize) {
                throw new ExodusException("Can't read expected high page bytes");
            }
            updatedTip = new LogTip(highPageContent, highPageAddress, highPageSize, highAddress, approvedHighAddress, fileSetImmutable);
        }
    }
    compareAndSetTip(logTip, updatedTip);
    this.bufferedWriter = null;
    return updatedTip;
}