Examples with ITupleReference org.apache.hyracks.dataflow.common.data.accessors.ITupleReference used on opensource projects

Example 51 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class AbstractRTreeTestDriver method rtreeTwoDimensionsInt.

@Test
public void rtreeTwoDimensionsInt() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("RTree " + getTestOpName() + " Test With Two Dimensions With Integer Keys.");
    }
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    int numKeys = 4;
    IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(numKeys, IntegerPointable.FACTORY);
    // Range search, the rectangle bottom left coordinates are -1000, -1000
    // and the top right coordinates are 1000, 1000
    ITupleReference key = TupleUtils.createIntegerTuple(-1000, -1000, 1000, 1000);
    runTest(fieldSerdes, valueProviderFactories, numKeys, key, RTreePolicyType.RTREE);
}

Example 52 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class ConnectorDescriptorWithMessagingTest method testMessageFitsWithTuples.

@Test
public void testMessageFitsWithTuples() throws Exception {
    try {
        // Routing will be round robin
        List<Integer> routing = Arrays.asList(0, 1, 2, 3, 4);
        IConnectorDescriptorRegistry connDescRegistry = Mockito.mock(IConnectorDescriptorRegistry.class);
        ITuplePartitionComputerFactory partitionComputerFactory = new TestPartitionComputerFactory(routing);
        MToNPartitioningWithMessageConnectorDescriptor connector = new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory);
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        VSizeFrame message = new VSizeFrame(ctx);
        VSizeFrame tempBuffer = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        message.getBuffer().clear();
        message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE);
        message.getBuffer().flip();
        ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
        FieldType[] types = { FieldType.Integer64, FieldType.Double, FieldType.Boolean, FieldType.String };
        RecordDescriptor rDesc = new RecordDescriptor(serdes);
        TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory();
        PartitionWithMessageDataWriter partitioner = (PartitionWithMessageDataWriter) connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS);
        partitioner.open();
        FrameTupleAccessor fta = new FrameTupleAccessor(rDesc);
        List<TestFrameWriter> recipients = new ArrayList<>();
        for (int i = 0; i < partitionWriterFactory.getWriters().values().size(); i++) {
            recipients.add(partitionWriterFactory.getWriters().get(i));
        }
        TestTupleGenerator ttg = new TestTupleGenerator(types, STRING_FIELD_SIZES, true);
        VSizeFrame frame = new VSizeFrame(ctx);
        FrameTupleAppender appender = new FrameTupleAppender(frame);
        for (int count = 0; count < NUMBER_OF_CONSUMERS; count++) {
            ITupleReference tuple = ttg.next();
            appender.append(tuple);
        }
        partitioner.nextFrame(frame.getBuffer());
        partitioner.flush();
        Assert.assertEquals(partitionWriterFactory.getWriters().get(0).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(1).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(2).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(3).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(4).nextFrameCount(), 1);
        for (TestFrameWriter writer : recipients) {
            fta.reset(writer.getLastFrame());
            Assert.assertEquals(fta.getTupleCount(), 2);
            FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
            Assert.assertEquals(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
        }
        partitioner.close();
    } catch (Throwable th) {
        th.printStackTrace();
        throw th;
    }
}

Example 53 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class LogMarkerTest method testInsertWithSnapshot.

@Test
public void testInsertWithSnapshot() {
    try {
        TestNodeController nc = new TestNodeController(null, false);
        nc.init();
        StorageComponentProvider storageManager = new StorageComponentProvider();
        List<List<String>> partitioningKeys = new ArrayList<>();
        partitioningKeys.add(Collections.singletonList("key"));
        Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
        try {
            nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            IHyracksTaskContext ctx = nc.createTestContext(true);
            nc.newJobId();
            ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
            LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager).getLeft();
            insertOp.open();
            TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
            VSizeFrame frame = new VSizeFrame(ctx);
            VSizeFrame marker = new VSizeFrame(ctx);
            FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
            long markerId = 0L;
            for (int j = 0; j < NUM_OF_RECORDS; j++) {
                if (j % SNAPSHOT_SIZE == 0) {
                    marker.reset();
                    marker.getBuffer().put(MessagingFrameTupleAppender.MARKER_MESSAGE);
                    marker.getBuffer().putLong(markerId);
                    marker.getBuffer().flip();
                    markerId++;
                    TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx);
                    tupleAppender.flush(insertOp);
                }
                ITupleReference tuple = tupleGenerator.next();
                DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
            }
            if (tupleAppender.getTupleCount() > 0) {
                tupleAppender.write(insertOp, true);
            }
            insertOp.close();
            nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
            IIndexDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            dataflowHelper.open();
            LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance();
            LongPointable longPointable = LongPointable.FACTORY.createPointable();
            ComponentMetadataUtil.get(btree, ComponentMetadataUtil.MARKER_LSN_KEY, longPointable);
            long lsn = longPointable.getLong();
            int numOfMarkers = 0;
            LogReader logReader = (LogReader) nc.getTransactionSubsystem().getLogManager().getLogReader(false);
            long expectedMarkerId = markerId - 1;
            while (lsn >= 0) {
                numOfMarkers++;
                ILogRecord logRecord = logReader.read(lsn);
                lsn = logRecord.getPreviousMarkerLSN();
                long logMarkerId = logRecord.getMarker().getLong();
                Assert.assertEquals(expectedMarkerId, logMarkerId);
                expectedMarkerId--;
            }
            logReader.close();
            dataflowHelper.close();
            Assert.assertEquals(markerId, numOfMarkers);
            nc.newJobId();
            TestTupleCounterFrameWriter countOp = create(nc.getSearchOutputDesc(KEY_TYPES, RECORD_TYPE, META_TYPE), Collections.emptyList(), Collections.emptyList(), false);
            IPushRuntime emptyTupleOp = nc.getFullScanPipeline(countOp, ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager);
            emptyTupleOp.open();
            emptyTupleOp.close();
            Assert.assertEquals(NUM_OF_RECORDS, countOp.getCount());
        } finally {
            nc.deInit();
        }
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 54 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class LSMInvertedIndex method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
    search(opCtx, cursor, mergePred);
    // Create an inverted index instance.
    LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(), mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader;
    // lsmHarness.endSearch() is called once when the inverted indexes have been merged.
    if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
        // Keep the deleted tuples since the oldest disk component is not included in the merge operation
        LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
        search(opCtx, btreeCursor, mergePred);
        long numElements = 0L;
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
        }
        componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
        try {
            while (btreeCursor.hasNext()) {
                btreeCursor.next();
                ITupleReference tuple = btreeCursor.getTuple();
                componentBulkLoader.delete(tuple);
            }
        } finally {
            btreeCursor.close();
        }
    } else {
        componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
    }
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference tuple = cursor.getTuple();
            componentBulkLoader.add(tuple);
        }
    } finally {
        cursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
            ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
            if (min != null) {
                filterTuples.add(min);
            }
            if (max != null) {
                filterTuples.add(max);
            }
        }
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
    }
    componentBulkLoader.end();
    return component;
}

Example 55 with ITupleReference

use of org.apache.hyracks.dataflow.common.data.accessors.ITupleReference in project asterixdb by apache.

the class LSMInvertedIndex method modify.

/**
     * The keys in the in-memory deleted-keys BTree only refer to on-disk components.
     * We delete documents from the in-memory inverted index by deleting its entries directly,
     * while still adding the deleted key to the deleted-keys BTree.
     * Otherwise, inserts would have to remove keys from the in-memory deleted-keys BTree which
     * may cause incorrect behavior (lost deletes) in the following pathological case:
     * Insert doc 1, flush, delete doc 1, insert doc 1
     * After the sequence above doc 1 will now appear twice because the delete of the on-disk doc 1 has been lost.
     * Insert:
     * - Insert document into in-memory inverted index.
     * Delete:
     * - Delete document from in-memory inverted index (ignore if it does not exist).
     * - Insert key into deleted-keys BTree.
     */
@Override
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException {
    LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx;
    // TODO: This is a hack to support logging properly in ASTERIX.
    // The proper undo operations are only dependent on the after image so
    // it is correct to say we found nothing (null) as the before image (at least
    // in the perspective of ASTERIX). The semantics for the operation callbacks
    // are violated here (and they are somewhat unclear in the first place as to
    // what they should be for an inverted index).
    ITupleReference indexTuple;
    if (ctx.getIndexTuple() != null) {
        ctx.getIndexTuple().reset(tuple);
        indexTuple = ctx.getIndexTuple();
    } else {
        indexTuple = tuple;
    }
    ctx.getModificationCallback().before(indexTuple);
    ctx.getModificationCallback().found(null, indexTuple);
    switch(ctx.getOperation()) {
        case INSERT:
            // Insert into the in-memory inverted index.
            ctx.getCurrentMutableInvIndexAccessors().insert(indexTuple);
            break;
        case DELETE:
            // First remove all entries in the in-memory inverted index (if any).
            ctx.getCurrentMutableInvIndexAccessors().delete(indexTuple);
            // Insert key into the deleted-keys BTree.
            ctx.getKeysOnlyTuple().reset(indexTuple);
            try {
                ctx.getCurrentDeletedKeysBTreeAccessors().insert(ctx.getKeysOnlyTuple());
            } catch (HyracksDataException e) {
                if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                    // Key has already been deleted.
                    LOGGER.log(Level.WARNING, "Failure during index delete operation", e);
                    throw e;
                }
            }
            break;
        default:
            throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
    }
    updateFilter(ctx, tuple);
}