Examples with FileReference org.apache.hyracks.api.io.FileReference used on opensource projects

Example 46 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class VirtualFreePageManagerTest method test01.

@Test
public void test01() throws HyracksDataException {
    VirtualBufferCache bufferCache = new VirtualBufferCache(new HeapBufferAllocator(), 4096, 128);
    bufferCache.open();
    FileReference fileRef = new FileReference(new IODeviceHandle(new File("target"), "workspace"), "tempfile.tmp");
    bufferCache.createFile(fileRef);
    int fileId = bufferCache.getFileMapProvider().lookupFileId(fileRef);
    bufferCache.openFile(fileId);
    VirtualFreePageManager virtualFreePageManager = new VirtualFreePageManager(bufferCache);
    virtualFreePageManager.open(fileId);
    virtualFreePageManager.init(null, null);
    testInMemoryFreePageManager(virtualFreePageManager);
    // We expect exactly the same behavior after a reset().
    virtualFreePageManager.init(null, null);
    testInMemoryFreePageManager(virtualFreePageManager);
}

Example 47 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class InvertedIndexUtils method createLSMInvertedIndex.

public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException {
    BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
    int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
    for (int i = 0; i < invListCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
    LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory);
    IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
    OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}

Example 48 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class InvertedIndexUtils method createOnDiskInvertedIndex.

public static OnDiskInvertedIndex createOnDiskInvertedIndex(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException {
    IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
    FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
    return new OnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
}

Example 49 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class InvertedIndexUtils method createPartitionedOnDiskInvertedIndex.

public static PartitionedOnDiskInvertedIndex createPartitionedOnDiskInvertedIndex(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, FileReference invListsFile, IPageManagerFactory pageManagerFactory) throws HyracksDataException {
    IInvertedListBuilder builder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
    FileReference btreeFile = getBTreeFile(ioManager, invListsFile);
    return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, builder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, btreeFile, invListsFile, pageManagerFactory);
}

Example 50 with FileReference

use of org.apache.hyracks.api.io.FileReference in project asterixdb by apache.

the class InvertedIndexUtils method createPartitionedLSMInvertedIndex.

public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IPageManagerFactory pageManagerFactory) throws HyracksDataException {
    BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
    int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
    for (int i = 0; i < invListCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
    LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory);
    IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
    PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}