Examples with IFileHandle org.apache.hyracks.api.io.IFileHandle used on opensource projects

Example 1 with IFileHandle

use of org.apache.hyracks.api.io.IFileHandle in project asterixdb by apache.

the class BufferCache method openFile.

@Override
public void openFile(int fileId) throws HyracksDataException {
    if (LOGGER.isLoggable(fileOpsLevel)) {
        LOGGER.log(fileOpsLevel, "Opening file: " + fileId + " in cache: " + this);
    }
    synchronized (fileInfoMap) {
        BufferedFileHandle fInfo;
        fInfo = fileInfoMap.get(fileId);
        if (fInfo == null) {
            // map is full, make room by cleaning up unreferenced files
            boolean unreferencedFileFound = true;
            while (fileInfoMap.size() >= maxOpenFiles && unreferencedFileFound) {
                unreferencedFileFound = false;
                for (Map.Entry<Integer, BufferedFileHandle> entry : fileInfoMap.entrySet()) {
                    if (entry.getValue().getReferenceCount() <= 0) {
                        int entryFileId = entry.getKey();
                        boolean fileHasBeenDeleted = entry.getValue().fileHasBeenDeleted();
                        sweepAndFlush(entryFileId, !fileHasBeenDeleted);
                        if (!fileHasBeenDeleted) {
                            ioManager.close(entry.getValue().getFileHandle());
                        }
                        fileInfoMap.remove(entryFileId);
                        unreferencedFileFound = true;
                        // fileInfoMap
                        break;
                    }
                }
            }
            if (fileInfoMap.size() >= maxOpenFiles) {
                throw new HyracksDataException("Could not open fileId " + fileId + ". Max number of files " + maxOpenFiles + " already opened and referenced.");
            }
            // create, open, and map new file reference
            FileReference fileRef = fileMapManager.lookupFileName(fileId);
            IFileHandle fh = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE, IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
            fInfo = new BufferedFileHandle(fileId, fh);
            fileInfoMap.put(fileId, fInfo);
        }
        fInfo.incReferenceCount();
    }
}

Example 2 with IFileHandle

use of org.apache.hyracks.api.io.IFileHandle in project asterixdb by apache.

the class IOManager method syncWrite.

@Override
public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException {
    try {
        if (fHandle == null) {
            throw new IllegalStateException("Trying to write to a deleted file.");
        }
        int n = 0;
        int remaining = 0;
        for (ByteBuffer buf : dataArray) {
            remaining += buf.remaining();
        }
        final FileChannel fileChannel = ((FileHandle) fHandle).getFileChannel();
        while (remaining > 0) {
            long len;
            synchronized (fileChannel) {
                fileChannel.position(offset);
                len = fileChannel.write(dataArray);
            }
            if (len < 0) {
                throw new HyracksDataException("Error writing to file: " + ((FileHandle) fHandle).getFileReference().toString());
            }
            remaining -= len;
            offset += len;
            n += len;
        }
        return n;
    } catch (HyracksDataException e) {
        throw e;
    } catch (IOException e) {
        throw new HyracksDataException(e);
    }
}

Example 3 with IFileHandle

use of org.apache.hyracks.api.io.IFileHandle in project asterixdb by apache.

the class BufferCacheRegressionTest method flushBehaviorTest.

private void flushBehaviorTest(boolean deleteFile) throws IOException {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, 10, 1);
    IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    IOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
    FileReference firstFileRef = ioManager.resolve(fileName);
    bufferCache.createFile(firstFileRef);
    int firstFileId = fmp.lookupFileId(firstFileRef);
    bufferCache.openFile(firstFileId);
    // Fill the first page with known data and make it dirty by write
    // latching it.
    ICachedPage writePage = bufferCache.pin(BufferedFileHandle.getDiskPageId(firstFileId, 0), true);
    writePage.acquireWriteLatch();
    try {
        ByteBuffer buf = writePage.getBuffer();
        for (int i = 0; i < buf.capacity(); i++) {
            buf.put(Byte.MAX_VALUE);
        }
    } finally {
        writePage.releaseWriteLatch(true);
        bufferCache.unpin(writePage);
    }
    bufferCache.closeFile(firstFileId);
    if (deleteFile) {
        bufferCache.deleteFile(firstFileId, false);
    }
    // Create a file with the same name.
    FileReference secondFileRef = ioManager.resolve(fileName);
    bufferCache.createFile(secondFileRef);
    int secondFileId = fmp.lookupFileId(secondFileRef);
    // This open will replace the firstFileRef's slot in the BufferCache,
    // causing it's pages to be cleaned up. We want to make sure that those
    // dirty pages are not flushed to the disk, because the file was
    // declared as deleted, and
    // somebody might be already using the same filename again (having been
    // assigned a different fileId).
    bufferCache.openFile(secondFileId);
    // Manually open the file and inspect it's contents. We cannot simply
    // ask the BufferCache to pin the page, because it would return the same
    // physical memory again, and for performance reasons pages are never
    // reset with 0's.
    FileReference testFileRef = ioManager.resolve(fileName);
    IFileHandle testFileHandle = ioManager.open(testFileRef, FileReadWriteMode.READ_ONLY, FileSyncMode.METADATA_SYNC_DATA_SYNC);
    ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE + BufferCache.RESERVED_HEADER_BYTES);
    ioManager.syncRead(testFileHandle, 0, testBuffer);
    for (int i = BufferCache.RESERVED_HEADER_BYTES; i < testBuffer.capacity(); i++) {
        if (deleteFile) {
            // We deleted the file. We expect to see a clean buffer.
            if (testBuffer.get(i) == Byte.MAX_VALUE) {
                fail("Page 0 of deleted file was fazily flushed in openFile(), " + "corrupting the data of a newly created file with the same name.");
            }
        } else {
            // Byte.MAX_VALUE.
            if (testBuffer.get(i) != Byte.MAX_VALUE) {
                fail("Page 0 of closed file was not flushed when properly, when reclaiming the file slot of fileId 0 in the BufferCache.");
            }
        }
    }
    ioManager.close(testFileHandle);
    bufferCache.closeFile(secondFileId);
    if (deleteFile) {
        bufferCache.deleteFile(secondFileId, false);
    }
    bufferCache.close();
}