Examples with IFrame org.apache.hyracks.api.comm.IFrame used on opensource projects

Example 21 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class RunMergingFrameReaderTest method testNormalRunMergeWithTopK.

@Test
public void testNormalRunMergeWithTopK() throws HyracksDataException {
    int pageSize = 128;
    int numRuns = 2;
    int numFramesPerRun = 2;
    int minRecordSize = pageSize / 10;
    int maxRecordSize = pageSize / 8;
    for (int topK = 1; topK < pageSize * numRuns * numFramesPerRun / maxRecordSize / 2; topK++) {
        IHyracksTaskContext ctx = testUtils.create(pageSize);
        List<Map<Integer, String>> keyValueMapList = new ArrayList<>(numRuns);
        List<TestFrameReader> readerList = new ArrayList<>(numRuns);
        List<IFrame> frameList = new ArrayList<>(numRuns);
        prepareRandomInputRunList(ctx, pageSize, numRuns, numFramesPerRun, minRecordSize, maxRecordSize, readerList, frameList, keyValueMapList);
        RunMergingFrameReader reader = new RunMergingFrameReader(ctx, readerList, frameList, SortFields, Comparators, null, RecordDesc, topK);
        int totoalCount = testMergeSucceedInner(ctx, reader, keyValueMapList);
        int newCount = 0;
        for (Map<Integer, String> x : keyValueMapList) {
            newCount += x.size();
        }
        assertEquals(topK + newCount, totoalCount);
    }
}

Example 22 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class BTreeStatsTest method test01.

@Test
public void test01() throws Exception {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = harness.getBufferCache();
    IFileMapProvider fmp = harness.getFileMapProvider();
    // declare fields
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // declare keys
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
    ITreeIndexMetadataFrame metaFrame = metaFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    Random rnd = new Random();
    rnd.setSeed(50);
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("INSERTING INTO TREE");
    }
    IFrame frame = new VSizeFrame(ctx);
    FrameTupleAppender appender = new FrameTupleAppender();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    DataOutput dos = tb.getDataOutput();
    ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
    IFrameTupleAccessor accessor = new FrameTupleAccessor(recDesc);
    accessor.reset(frame.getBuffer());
    FrameTupleReference tuple = new FrameTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // 10000
    for (int i = 0; i < 100000; i++) {
        int f0 = rnd.nextInt() % 100000;
        int f1 = 5;
        tb.reset();
        IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
        tb.addFieldEndOffset();
        IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
        tb.addFieldEndOffset();
        appender.reset(frame, true);
        appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
        tuple.reset(accessor, 0);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 10000 == 0) {
                long end = System.currentTimeMillis();
                LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                e.printStackTrace();
                throw e;
            }
        }
    }
    int fileId = fmp.lookupFileId(harness.getFileReference());
    TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId, btree.getRootPageId());
    TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("\n" + stats.toString());
    }
    TreeIndexBufferCacheWarmup bufferCacheWarmup = new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, fileId);
    bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 });
    btree.deactivate();
    btree.destroy();
    bufferCache.close();
}

Example 23 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class FieldPrefixNSMTest method createTuple.

private ITupleReference createTuple(IHyracksTaskContext ctx, int f0, int f1, int f2, boolean print) throws HyracksDataException {
    if (print) {
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("CREATING: " + f0 + " " + f1 + " " + f2);
        }
    }
    IFrame buf = new VSizeFrame(ctx);
    FrameTupleAppender appender = new FrameTupleAppender(buf);
    ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
    DataOutput dos = tb.getDataOutput();
    @SuppressWarnings("rawtypes") ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
    IFrameTupleAccessor accessor = new FrameTupleAccessor(recDesc);
    accessor.reset(buf.getBuffer());
    FrameTupleReference tuple = new FrameTupleReference();
    tb.reset();
    IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
    tb.addFieldEndOffset();
    IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
    tb.addFieldEndOffset();
    IntegerSerializerDeserializer.INSTANCE.serialize(f2, dos);
    tb.addFieldEndOffset();
    appender.reset(buf, true);
    appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
    tuple.reset(accessor, 0);
    return tuple;
}

Example 24 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class FeedFrameUtil method getSampledFrame.

public static ByteBuffer getSampledFrame(IHyracksTaskContext ctx, FrameTupleAccessor fta, int sampleSize) throws HyracksDataException {
    NChooseKIterator it = new NChooseKIterator(fta.getTupleCount(), sampleSize);
    FrameTupleAppender appender = new FrameTupleAppender();
    IFrame sampledFrame = new VSizeFrame(ctx);
    appender.reset(sampledFrame, true);
    int nextTupleIndex = 0;
    while (it.hasNext()) {
        nextTupleIndex = it.next();
        appender.append(fta, nextTupleIndex);
    }
    return sampledFrame.getBuffer();
}

Example 25 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class FeedIntakeOperatorNodePushable method start.

@Override
protected void start() throws HyracksDataException, InterruptedException {
    try {
        writer.open();
        Thread.currentThread().setName("Intake Thread");
        FeedAdapter adapter = (FeedAdapter) adapterFactory.createAdapter(ctx, partition);
        adapterRuntimeManager = new AdapterRuntimeManager(ctx, runtimeId.getEntityId(), adapter, writer, partition);
        IFrame message = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        /*
             * Set null feed message. Feed pipeline carries with it a message with each frame
             * Initially, the message is set to a null message that can be changed by feed adapters.
             * One use case is adapters which consume data sources that allow restartability. Such adapters
             * can propagate progress information through the ingestion pipeline to storage nodes
             */
        message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
        message.getBuffer().flip();
        adapterRuntimeManager.start();
        synchronized (adapterRuntimeManager) {
            while (!adapterRuntimeManager.isDone()) {
                adapterRuntimeManager.wait();
            }
        }
        if (adapterRuntimeManager.isFailed()) {
            throw new RuntimeDataException(ErrorCode.OPERATORS_FEED_INTAKE_OPERATOR_NODE_PUSHABLE_FAIL_AT_INGESTION);
        }
    } catch (Exception e) {
        /*
             * An Interrupted Exception is thrown if the Intake job cannot progress further due to failure of another
             * node involved in the Hyracks job. As the Intake job involves only the intake operator, the exception is
             * indicative of a failure at the sibling intake operator location. The surviving intake partitions must
             * continue to live and receive data from the external source.
             */
        writer.fail();
        throw e;
    } finally {
        writer.close();
    }
}