Search in sources :

Example 11 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class ConnectorDescriptorWithMessagingTest method testEmptyFrames.

@Test
public void testEmptyFrames() throws Exception {
    try {
        List<Integer> routing = Arrays.asList(0, 1, 2, 3, 4);
        IConnectorDescriptorRegistry connDescRegistry = Mockito.mock(IConnectorDescriptorRegistry.class);
        ITuplePartitionComputerFactory partitionComputerFactory = new TestPartitionComputerFactory(routing);
        MToNPartitioningWithMessageConnectorDescriptor connector = new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory);
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        VSizeFrame message = new VSizeFrame(ctx);
        VSizeFrame tempBuffer = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        message.getBuffer().clear();
        message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
        message.getBuffer().flip();
        ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
        RecordDescriptor rDesc = new RecordDescriptor(serdes);
        TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory();
        IFrameWriter partitioner = connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS);
        List<TestFrameWriter> recipients = new ArrayList<>();
        try {
            partitioner.open();
            FrameTupleAccessor fta = new FrameTupleAccessor(rDesc);
            for (IFrameWriter writer : partitionWriterFactory.getWriters().values()) {
                recipients.add((TestFrameWriter) writer);
            }
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 1);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.NULL_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
            message.getBuffer().clear();
            message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE);
            message.getBuffer().flip();
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 2);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
            message.getBuffer().clear();
            message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
            message.getBuffer().flip();
            partitioner.flush();
            for (TestFrameWriter writer : recipients) {
                Assert.assertEquals(writer.nextFrameCount(), 3);
                fta.reset(writer.getLastFrame());
                Assert.assertEquals(fta.getTupleCount(), 1);
                FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
                Assert.assertEquals(MessagingFrameTupleAppender.NULL_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
            }
        } catch (Throwable t) {
            partitioner.fail();
            throw t;
        } finally {
            partitioner.close();
        }
        for (TestFrameWriter writer : recipients) {
            Assert.assertEquals(writer.nextFrameCount(), 4);
            Assert.assertEquals(writer.closeCount(), 1);
        }
    } catch (Throwable th) {
        th.printStackTrace();
        throw th;
    }
}
Also used : ITuplePartitionComputerFactory(org.apache.hyracks.api.dataflow.value.ITuplePartitionComputerFactory) MToNPartitioningWithMessageConnectorDescriptor(org.apache.hyracks.dataflow.std.connectors.MToNPartitioningWithMessageConnectorDescriptor) IFrameWriter(org.apache.hyracks.api.comm.IFrameWriter) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) TestFrameWriter(org.apache.hyracks.api.test.TestFrameWriter) ArrayList(java.util.ArrayList) UTF8StringSerializerDeserializer(org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame) ISerializerDeserializer(org.apache.hyracks.api.dataflow.value.ISerializerDeserializer) IConnectorDescriptorRegistry(org.apache.hyracks.api.job.IConnectorDescriptorRegistry) IHyracksTaskContext(org.apache.hyracks.api.context.IHyracksTaskContext) FrameTupleAccessor(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor) Test(org.junit.Test)

Example 12 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class ConnectorDescriptorWithMessagingTest method testMessageFitsWithTuples.

@Test
public void testMessageFitsWithTuples() throws Exception {
    try {
        // Routing will be round robin
        List<Integer> routing = Arrays.asList(0, 1, 2, 3, 4);
        IConnectorDescriptorRegistry connDescRegistry = Mockito.mock(IConnectorDescriptorRegistry.class);
        ITuplePartitionComputerFactory partitionComputerFactory = new TestPartitionComputerFactory(routing);
        MToNPartitioningWithMessageConnectorDescriptor connector = new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory);
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        VSizeFrame message = new VSizeFrame(ctx);
        VSizeFrame tempBuffer = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        message.getBuffer().clear();
        message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE);
        message.getBuffer().flip();
        ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
        FieldType[] types = { FieldType.Integer64, FieldType.Double, FieldType.Boolean, FieldType.String };
        RecordDescriptor rDesc = new RecordDescriptor(serdes);
        TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory();
        PartitionWithMessageDataWriter partitioner = (PartitionWithMessageDataWriter) connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS);
        partitioner.open();
        FrameTupleAccessor fta = new FrameTupleAccessor(rDesc);
        List<TestFrameWriter> recipients = new ArrayList<>();
        for (int i = 0; i < partitionWriterFactory.getWriters().values().size(); i++) {
            recipients.add(partitionWriterFactory.getWriters().get(i));
        }
        TestTupleGenerator ttg = new TestTupleGenerator(types, STRING_FIELD_SIZES, true);
        VSizeFrame frame = new VSizeFrame(ctx);
        FrameTupleAppender appender = new FrameTupleAppender(frame);
        for (int count = 0; count < NUMBER_OF_CONSUMERS; count++) {
            ITupleReference tuple = ttg.next();
            appender.append(tuple);
        }
        partitioner.nextFrame(frame.getBuffer());
        partitioner.flush();
        Assert.assertEquals(partitionWriterFactory.getWriters().get(0).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(1).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(2).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(3).nextFrameCount(), 1);
        Assert.assertEquals(partitionWriterFactory.getWriters().get(4).nextFrameCount(), 1);
        for (TestFrameWriter writer : recipients) {
            fta.reset(writer.getLastFrame());
            Assert.assertEquals(fta.getTupleCount(), 2);
            FeedUtils.processFeedMessage(writer.getLastFrame(), tempBuffer, fta);
            Assert.assertEquals(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE, MessagingFrameTupleAppender.getMessageType(tempBuffer));
        }
        partitioner.close();
    } catch (Throwable th) {
        th.printStackTrace();
        throw th;
    }
}
Also used : PartitionWithMessageDataWriter(org.apache.hyracks.dataflow.std.connectors.PartitionWithMessageDataWriter) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) TestFrameWriter(org.apache.hyracks.api.test.TestFrameWriter) ArrayList(java.util.ArrayList) TestTupleGenerator(org.apache.asterix.test.common.TestTupleGenerator) UTF8StringSerializerDeserializer(org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer) IConnectorDescriptorRegistry(org.apache.hyracks.api.job.IConnectorDescriptorRegistry) MessagingFrameTupleAppender(org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender) FrameTupleAppender(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender) ITuplePartitionComputerFactory(org.apache.hyracks.api.dataflow.value.ITuplePartitionComputerFactory) MToNPartitioningWithMessageConnectorDescriptor(org.apache.hyracks.dataflow.std.connectors.MToNPartitioningWithMessageConnectorDescriptor) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame) ISerializerDeserializer(org.apache.hyracks.api.dataflow.value.ISerializerDeserializer) FieldType(org.apache.asterix.test.common.TestTupleGenerator.FieldType) IHyracksTaskContext(org.apache.hyracks.api.context.IHyracksTaskContext) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) FrameTupleAccessor(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor) Test(org.junit.Test)

Example 13 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class LogMarkerTest method testInsertWithSnapshot.

@Test
public void testInsertWithSnapshot() {
    try {
        TestNodeController nc = new TestNodeController(null, false);
        nc.init();
        StorageComponentProvider storageManager = new StorageComponentProvider();
        List<List<String>> partitioningKeys = new ArrayList<>();
        partitioningKeys.add(Collections.singletonList("key"));
        Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
        try {
            nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            IHyracksTaskContext ctx = nc.createTestContext(true);
            nc.newJobId();
            ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
            LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager).getLeft();
            insertOp.open();
            TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
            VSizeFrame frame = new VSizeFrame(ctx);
            VSizeFrame marker = new VSizeFrame(ctx);
            FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
            long markerId = 0L;
            for (int j = 0; j < NUM_OF_RECORDS; j++) {
                if (j % SNAPSHOT_SIZE == 0) {
                    marker.reset();
                    marker.getBuffer().put(MessagingFrameTupleAppender.MARKER_MESSAGE);
                    marker.getBuffer().putLong(markerId);
                    marker.getBuffer().flip();
                    markerId++;
                    TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx);
                    tupleAppender.flush(insertOp);
                }
                ITupleReference tuple = tupleGenerator.next();
                DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
            }
            if (tupleAppender.getTupleCount() > 0) {
                tupleAppender.write(insertOp, true);
            }
            insertOp.close();
            nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
            IIndexDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            dataflowHelper.open();
            LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance();
            LongPointable longPointable = LongPointable.FACTORY.createPointable();
            ComponentMetadataUtil.get(btree, ComponentMetadataUtil.MARKER_LSN_KEY, longPointable);
            long lsn = longPointable.getLong();
            int numOfMarkers = 0;
            LogReader logReader = (LogReader) nc.getTransactionSubsystem().getLogManager().getLogReader(false);
            long expectedMarkerId = markerId - 1;
            while (lsn >= 0) {
                numOfMarkers++;
                ILogRecord logRecord = logReader.read(lsn);
                lsn = logRecord.getPreviousMarkerLSN();
                long logMarkerId = logRecord.getMarker().getLong();
                Assert.assertEquals(expectedMarkerId, logMarkerId);
                expectedMarkerId--;
            }
            logReader.close();
            dataflowHelper.close();
            Assert.assertEquals(markerId, numOfMarkers);
            nc.newJobId();
            TestTupleCounterFrameWriter countOp = create(nc.getSearchOutputDesc(KEY_TYPES, RECORD_TYPE, META_TYPE), Collections.emptyList(), Collections.emptyList(), false);
            IPushRuntime emptyTupleOp = nc.getFullScanPipeline(countOp, ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager);
            emptyTupleOp.open();
            emptyTupleOp.close();
            Assert.assertEquals(NUM_OF_RECORDS, countOp.getCount());
        } finally {
            nc.deInit();
        }
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}
Also used : LSMInsertDeleteOperatorNodePushable(org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable) IIndexDataflowHelper(org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper) IPushRuntime(org.apache.hyracks.algebricks.runtime.base.IPushRuntime) ArrayList(java.util.ArrayList) TestTupleCounterFrameWriter(org.apache.asterix.app.data.gen.TestTupleCounterFrameWriter) NoMergePolicyFactory(org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory) MessagingFrameTupleAppender(org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender) FrameTupleAppender(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender) LongPointable(org.apache.hyracks.data.std.primitive.LongPointable) ArrayList(java.util.ArrayList) List(java.util.List) ILogRecord(org.apache.asterix.common.transactions.ILogRecord) Dataset(org.apache.asterix.metadata.entities.Dataset) InternalDatasetDetails(org.apache.asterix.metadata.entities.InternalDatasetDetails) ITransactionContext(org.apache.asterix.common.transactions.ITransactionContext) TupleGenerator(org.apache.asterix.app.data.gen.TupleGenerator) StorageComponentProvider(org.apache.asterix.file.StorageComponentProvider) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame) IHyracksTaskContext(org.apache.hyracks.api.context.IHyracksTaskContext) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) TestNodeController(org.apache.asterix.app.bootstrap.TestNodeController) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) LogReader(org.apache.asterix.transaction.management.service.logging.LogReader) Test(org.junit.Test)

Example 14 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class ExternalRTreeSearchOperatorNodePushable method open.

// We override this method to specify the searched version of the index
@Override
public void open() throws HyracksDataException {
    writer.open();
    accessor = new FrameTupleAccessor(inputRecDesc);
    indexHelper.open();
    index = indexHelper.getIndexInstance();
    if (retainMissing) {
        int fieldCount = getFieldCount();
        nonMatchTupleBuild = new ArrayTupleBuilder(fieldCount);
        DataOutput out = nonMatchTupleBuild.getDataOutput();
        for (int i = 0; i < fieldCount; i++) {
            try {
                nonMatchWriter.writeMissing(out);
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
            nonMatchTupleBuild.addFieldEndOffset();
        }
    } else {
        nonMatchTupleBuild = null;
    }
    ExternalRTree rTreeIndex = (ExternalRTree) index;
    try {
        searchPred = createSearchPredicate();
        tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
        dos = tb.getDataOutput();
        appender = new FrameTupleAppender(new VSizeFrame(ctx));
        ISearchOperationCallback searchCallback = searchCallbackFactory.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
        // The next line is the reason we override this method...
        // The right thing to do would be to change the signature of createAccessor
        indexAccessor = rTreeIndex.createAccessor(searchCallback, version);
        cursor = createCursor();
        if (retainInput) {
            frameTuple = new FrameTupleReference();
        }
    } catch (Exception e) {
        throw new HyracksDataException(e);
    }
}
Also used : DataOutput(java.io.DataOutput) FrameTupleAppender(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender) FrameTupleReference(org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference) ArrayTupleBuilder(org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder) IOException(java.io.IOException) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) ISearchOperationCallback(org.apache.hyracks.storage.common.ISearchOperationCallback) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) IOException(java.io.IOException) FrameTupleAccessor(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor)

Example 15 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class RunMergingFrameReaderTest method testMergeSucceedInner.

private int testMergeSucceedInner(IHyracksTaskContext ctx, RunMergingFrameReader reader, List<Map<Integer, String>> keyValueMapList) throws HyracksDataException {
    IFrame frame = new VSizeFrame(ctx);
    reader.open();
    int count = 0;
    for (int i = 0; i < keyValueMapList.size(); i++) {
        keyValueMapList.set(i, new TreeMap<>(keyValueMapList.get(i)));
        count += keyValueMapList.get(i).size();
    }
    while (reader.nextFrame(frame)) {
        assertFrameIsSorted(frame, keyValueMapList);
    }
    return count;
}
Also used : IFrame(org.apache.hyracks.api.comm.IFrame) GroupVSizeFrame(org.apache.hyracks.dataflow.std.sort.util.GroupVSizeFrame) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame)

Aggregations

VSizeFrame (org.apache.hyracks.api.comm.VSizeFrame)63 FrameTupleAppender (org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender)32 HyracksDataException (org.apache.hyracks.api.exceptions.HyracksDataException)22 FrameTupleAccessor (org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor)18 IHyracksTaskContext (org.apache.hyracks.api.context.IHyracksTaskContext)17 ArrayTupleBuilder (org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder)16 Test (org.junit.Test)16 IFrame (org.apache.hyracks.api.comm.IFrame)13 ArrayList (java.util.ArrayList)11 DataOutput (java.io.DataOutput)10 IFrameTupleAccessor (org.apache.hyracks.api.comm.IFrameTupleAccessor)9 FrameTupleReference (org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference)9 IOException (java.io.IOException)8 IFrameWriter (org.apache.hyracks.api.comm.IFrameWriter)8 RecordDescriptor (org.apache.hyracks.api.dataflow.value.RecordDescriptor)8 ConcurrentFramePool (org.apache.asterix.common.memory.ConcurrentFramePool)7 FeedRuntimeInputHandler (org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler)7 FeedPolicyAccessor (org.apache.asterix.external.feed.policy.FeedPolicyAccessor)7 TestFrameWriter (org.apache.hyracks.api.test.TestFrameWriter)7 ISerializerDeserializer (org.apache.hyracks.api.dataflow.value.ISerializerDeserializer)6