Examples with IIOManager org.apache.hyracks.api.io.IIOManager used on opensource projects

Example 16 with IIOManager

use of org.apache.hyracks.api.io.IIOManager in project asterixdb by apache.

the class ExternalRTreeLocalResource method createInstance.

@Override
public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException {
    IIOManager ioManager = ncServiceCtx.getIoManager();
    FileReference fileRef = ioManager.resolve(path);
    return LSMRTreeUtils.createExternalRTree(ioManager, fileRef, storageManager.getBufferCache(ncServiceCtx), storageManager.getFileMapProvider(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx), opTrackerProvider.getOperationTracker(ncServiceCtx), ioSchedulerProvider.getIoScheduler(ncServiceCtx), ioOpCallbackFactory.createIoOpCallback(), linearizeCmpFactory, buddyBTreeFields, durable, isPointMBR, metadataPageManagerFactory);
}

Example 17 with IIOManager

use of org.apache.hyracks.api.io.IIOManager in project asterixdb by apache.

the class BufferCacheTest method simpleMaxOpenFilesTest.

@Test
public void simpleMaxOpenFilesTest() throws HyracksException {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
    List<Integer> fileIds = new ArrayList<>();
    for (int i = 0; i < MAX_OPEN_FILES; i++) {
        String fileName = getFileName();
        FileReference file = ioManager.resolve(fileName);
        bufferCache.createFile(file);
        int fileId = fmp.lookupFileId(file);
        bufferCache.openFile(fileId);
        fileIds.add(fileId);
    }
    boolean exceptionThrown = false;
    // since all files are open, next open should fail
    try {
        String fileName = getFileName();
        FileReference file = ioManager.resolve(fileName);
        bufferCache.createFile(file);
        int fileId = fmp.lookupFileId(file);
        bufferCache.openFile(fileId);
    } catch (HyracksDataException e) {
        exceptionThrown = true;
    }
    Assert.assertTrue(exceptionThrown);
    // close a random file
    int ix = Math.abs(rnd.nextInt()) % fileIds.size();
    bufferCache.closeFile(fileIds.get(ix));
    fileIds.remove(ix);
    // now open should succeed again
    exceptionThrown = false;
    try {
        String fileName = getFileName();
        FileReference file = ioManager.resolve(fileName);
        bufferCache.createFile(file);
        int fileId = fmp.lookupFileId(file);
        bufferCache.openFile(fileId);
        fileIds.add(fileId);
    } catch (HyracksDataException e) {
        exceptionThrown = true;
    }
    Assert.assertFalse(exceptionThrown);
    for (Integer i : fileIds) {
        bufferCache.closeFile(i.intValue());
    }
    bufferCache.close();
}

Example 18 with IIOManager

use of org.apache.hyracks.api.io.IIOManager in project asterixdb by apache.

the class BufferCacheTest method contentCheckingMaxOpenFilesTest.

@Test
public void contentCheckingMaxOpenFilesTest() throws HyracksException {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext());
    IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider();
    IIOManager ioManager = TestStorageManagerComponentHolder.getIOManager();
    List<Integer> fileIds = new ArrayList<>();
    Map<Integer, ArrayList<Integer>> pageContents = new HashMap<>();
    int num = 10;
    int testPageId = 0;
    // open max number of files and write some stuff into their first page
    for (int i = 0; i < MAX_OPEN_FILES; i++) {
        String fileName = getFileName();
        FileReference file = ioManager.resolve(fileName);
        bufferCache.createFile(file);
        int fileId = fmp.lookupFileId(file);
        bufferCache.openFile(fileId);
        fileIds.add(fileId);
        ICachedPage page = null;
        page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, testPageId), true);
        page.acquireWriteLatch();
        try {
            ArrayList<Integer> values = new ArrayList<>();
            for (int j = 0; j < num; j++) {
                int x = Math.abs(rnd.nextInt());
                page.getBuffer().putInt(j * 4, x);
                values.add(x);
            }
            pageContents.put(fileId, values);
        } finally {
            page.releaseWriteLatch(true);
            bufferCache.unpin(page);
        }
    }
    boolean exceptionThrown = false;
    // since all files are open, next open should fail
    try {
        String fileName = getFileName();
        FileReference file = ioManager.resolve(fileName);
        bufferCache.createFile(file);
        int fileId = fmp.lookupFileId(file);
        bufferCache.openFile(fileId);
    } catch (HyracksDataException e) {
        exceptionThrown = true;
    }
    Assert.assertTrue(exceptionThrown);
    // close a few random files
    ArrayList<Integer> closedFileIds = new ArrayList<>();
    int filesToClose = 5;
    for (int i = 0; i < filesToClose; i++) {
        int ix = Math.abs(rnd.nextInt()) % fileIds.size();
        bufferCache.closeFile(fileIds.get(ix));
        closedFileIds.add(fileIds.get(ix));
        fileIds.remove(ix);
    }
    // now open a few new files
    for (int i = 0; i < filesToClose; i++) {
        String fileName = getFileName();
        FileReference file = ioManager.resolve(fileName);
        bufferCache.createFile(file);
        int fileId = fmp.lookupFileId(file);
        bufferCache.openFile(fileId);
        fileIds.add(fileId);
    }
    // since all files are open, next open should fail
    try {
        String fileName = getFileName();
        FileReference file = ioManager.resolve(fileName);
        bufferCache.createFile(file);
        int fileId = fmp.lookupFileId(file);
        bufferCache.openFile(fileId);
    } catch (HyracksDataException e) {
        exceptionThrown = true;
    }
    Assert.assertTrue(exceptionThrown);
    // close a few random files again
    for (int i = 0; i < filesToClose; i++) {
        int ix = Math.abs(rnd.nextInt()) % fileIds.size();
        bufferCache.closeFile(fileIds.get(ix));
        closedFileIds.add(fileIds.get(ix));
        fileIds.remove(ix);
    }
    // now open those closed files again and verify their contents
    for (int i = 0; i < filesToClose; i++) {
        int closedFileId = closedFileIds.get(i);
        bufferCache.openFile(closedFileId);
        fileIds.add(closedFileId);
        // pin first page and verify contents
        ICachedPage page = null;
        page = bufferCache.pin(BufferedFileHandle.getDiskPageId(closedFileId, testPageId), false);
        page.acquireReadLatch();
        try {
            ArrayList<Integer> values = pageContents.get(closedFileId);
            for (int j = 0; j < values.size(); j++) {
                Assert.assertEquals(values.get(j).intValue(), page.getBuffer().getInt(j * 4));
            }
        } finally {
            page.releaseReadLatch();
            bufferCache.unpin(page);
        }
    }
    for (Integer i : fileIds) {
        bufferCache.closeFile(i.intValue());
    }
    bufferCache.close();
}

Example 19 with IIOManager

use of org.apache.hyracks.api.io.IIOManager in project asterixdb by apache.

the class ExternalDatasetIndexesAbortOperatorDescriptor method performOpOnIndex.

@Override
protected void performOpOnIndex(IIndexDataflowHelper indexDataflowHelper, IHyracksTaskContext ctx) throws HyracksDataException {
    String path = indexDataflowHelper.getResource().getPath();
    IIOManager ioManager = ctx.getIoManager();
    FileReference file = ioManager.resolve(path);
    AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(ctx.getIoManager(), file);
    fileManager.deleteTransactionFiles();
}

Example 20 with IIOManager

use of org.apache.hyracks.api.io.IIOManager in project asterixdb by apache.

the class FileRemoveOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    final FileSplit split = fileSplitProvider.getFileSplits()[partition];
    final IIOManager ioManager = ctx.getIoManager();
    return new AbstractOperatorNodePushable() {

        @Override
        public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
            throw new IllegalStateException();
        }

        @Override
        public void initialize() throws HyracksDataException {
            // will only work for files inside the io devices
            File f = split.getFile(ioManager);
            if (quietly) {
                FileUtils.deleteQuietly(f);
            } else {
                try {
                    FileUtils.deleteDirectory(f);
                } catch (IOException e) {
                    throw new HyracksDataException(e);
                }
            }
        }

        @Override
        public IFrameWriter getInputFrameWriter(int index) {
            throw new IllegalStateException();
        }

        @Override
        public int getInputArity() {
            return 0;
        }

        @Override
        public void deinitialize() throws HyracksDataException {
        }
    };
}