Examples with ITupleReference org.apache.hyracks.dataflow.common.data.accessors.ITupleReference used on opensource projects

Example 31 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class ExternalFileIndexAccessor method lookup.

public void lookup(int fileId, ExternalFile file) throws HyracksDataException {
    // Set search parameters
    currentFileNumber.setValue(fileId);
    searchKeyTupleBuilder.reset();
    searchKeyTupleBuilder.addField(intSerde, currentFileNumber);
    searchKey.reset(searchKeyTupleBuilder.getFieldEndOffsets(), searchKeyTupleBuilder.getByteArray());
    fileIndexSearchCursor.reset();
    // Perform search
    fileIndexAccessor.search(fileIndexSearchCursor, searchPredicate);
    if (fileIndexSearchCursor.hasNext()) {
        fileIndexSearchCursor.next();
        ITupleReference tuple = fileIndexSearchCursor.getTuple();
        // Deserialize
        byte[] serRecord = tuple.getFieldData(FilesIndexDescription.FILE_PAYLOAD_INDEX);
        int recordStartOffset = tuple.getFieldStart(FilesIndexDescription.FILE_PAYLOAD_INDEX);
        int recordLength = tuple.getFieldLength(FilesIndexDescription.FILE_PAYLOAD_INDEX);
        ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
        DataInput in = new DataInputStream(stream);
        ARecord externalFileRecord = (ARecord) externalFileRecordSerde.deserialize(in);
        setFile(externalFileRecord, file);
    } else {
        // This should never happen
        throw new RuntimeDataException(ErrorCode.INDEXING_EXTERNAL_FILE_INDEX_ACCESSOR_UNABLE_TO_FIND_FILE_INDEX);
    }
}

Example 32 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class IndexTupleTranslatorTest method test.

@Test
public void test() throws MetadataException, IOException {
    Integer[] indicators = { 0, 1, null };
    for (Integer indicator : indicators) {
        Map<String, String> compactionPolicyProperties = new HashMap<>();
        compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
        compactionPolicyProperties.put("max-tolerance-component-count", "3");
        InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH, Collections.singletonList(Collections.singletonList("row_id")), Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
        Dataset dataset = new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES", "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115, 0);
        Index index = new Index("test", "d1", "i1", IndexType.BTREE, Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), Collections.singletonList(BuiltinType.AINT64), -1, false, false, 0);
        MetadataNode mockMetadataNode = mock(MetadataNode.class);
        when(mockMetadataNode.getDatatype(any(), anyString(), anyString())).thenReturn(new Datatype("test", "d1", new ARecordType("", new String[] { "row_id" }, new IAType[] { BuiltinType.AINT64 }, true), true));
        when(mockMetadataNode.getDataset(any(), anyString(), anyString())).thenReturn(dataset);
        IndexTupleTranslator idxTranslator = new IndexTupleTranslator(null, mockMetadataNode, true);
        ITupleReference tuple = idxTranslator.getTupleFromMetadataEntity(index);
        Index deserializedIndex = idxTranslator.getMetadataEntityFromTuple(tuple);
        if (indicator == null) {
            Assert.assertEquals(Collections.singletonList(new Integer(0)), deserializedIndex.getKeyFieldSourceIndicators());
        } else {
            Assert.assertEquals(index.getKeyFieldSourceIndicators(), deserializedIndex.getKeyFieldSourceIndicators());
        }
    }
}

Example 33 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class LSMRTreeWithAntiMatterTuples method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    MergeOperation mergeOp = (MergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rtreeSearchPred);
    // Bulk load the tuples from all on-disk RTrees into the new RTree.
    LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
    }
    componentBulkLoader.end();
    return component;
}

Example 34 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class LSMRTreeWithAntiMatterTuplesSearchCursor method hasNext.

@Override
public boolean hasNext() throws HyracksDataException {
    if (includeMutableComponent) {
        if (foundNext) {
            return true;
        }
        while (currentCursor < numMutableComponents) {
            while (mutableRTreeCursors[currentCursor].hasNext()) {
                mutableRTreeCursors[currentCursor].next();
                ITupleReference currentTuple = mutableRTreeCursors[currentCursor].getTuple();
                // TODO: at this time, we only add proceed() part.
                // reconcile() and complete() can be added later after considering the semantics.
                // Call proceed() to do necessary operations before returning this tuple.
                searchCallback.proceed(currentTuple);
                if (searchMemBTrees(currentTuple, currentCursor)) {
                    // anti-matter tuple is NOT found
                    foundNext = true;
                    frameTuple = currentTuple;
                    return true;
                }
            }
            mutableRTreeCursors[currentCursor].close();
            currentCursor++;
            searchNextCursor();
        }
        while (super.hasNext()) {
            super.next();
            ITupleReference diskRTreeTuple = super.getTuple();
            // TODO: at this time, we only add proceed().
            // reconcile() and complete() can be added later after considering the semantics.
            // Call proceed() to do necessary operations before returning this tuple.
            searchCallback.proceed(diskRTreeTuple);
            if (searchMemBTrees(diskRTreeTuple, numMutableComponents)) {
                // anti-matter tuple is NOT found
                foundNext = true;
                frameTuple = diskRTreeTuple;
                return true;
            }
        }
    } else {
        if (super.hasNext()) {
            super.next();
            ITupleReference diskRTreeTuple = super.getTuple();
            // TODO: at this time, we only add proceed() part.
            // reconcile() and complete() can be added later after considering the semantics.
            // Call proceed() to do necessary operations before returning this tuple.
            // Since in-memory components don't exist, we can skip searching in-memory B-Trees.
            searchCallback.proceed(diskRTreeTuple);
            foundNext = true;
            frameTuple = diskRTreeTuple;
            return true;
        }
    }
    return false;
}

Example 35 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class LSMRTree method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
    ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rtreeSearchPred);
    LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader;
    // lsmHarness.endSearch() is called once when the r-trees have been merged.
    if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
        // Keep the deleted tuples since the oldest disk component is not included in the merge operation
        long numElements = 0L;
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
        }
        componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
        LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx);
        search(opCtx, btreeCursor, rtreeSearchPred);
        try {
            while (btreeCursor.hasNext()) {
                btreeCursor.next();
                ITupleReference tuple = btreeCursor.getTuple();
                componentBulkLoader.delete(tuple);
            }
        } finally {
            btreeCursor.close();
        }
    } else {
        //no buddy-btree needed
        componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false);
    }
    //search old rtree components
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (mergedComponent.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
    }
    componentBulkLoader.end();
    return mergedComponent;
}