Examples with VSizeFrame org.apache.hyracks.api.comm.VSizeFrame used on opensource projects

Example 46 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class BTreeStatsTest method test01.

@Test
public void test01() throws Exception {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = harness.getBufferCache();
    IFileMapProvider fmp = harness.getFileMapProvider();
    // declare fields
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // declare keys
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
    ITreeIndexMetadataFrame metaFrame = metaFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    Random rnd = new Random();
    rnd.setSeed(50);
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("INSERTING INTO TREE");
    }
    IFrame frame = new VSizeFrame(ctx);
    FrameTupleAppender appender = new FrameTupleAppender();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    DataOutput dos = tb.getDataOutput();
    ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
    IFrameTupleAccessor accessor = new FrameTupleAccessor(recDesc);
    accessor.reset(frame.getBuffer());
    FrameTupleReference tuple = new FrameTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // 10000
    for (int i = 0; i < 100000; i++) {
        int f0 = rnd.nextInt() % 100000;
        int f1 = 5;
        tb.reset();
        IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
        tb.addFieldEndOffset();
        IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
        tb.addFieldEndOffset();
        appender.reset(frame, true);
        appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
        tuple.reset(accessor, 0);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 10000 == 0) {
                long end = System.currentTimeMillis();
                LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                e.printStackTrace();
                throw e;
            }
        }
    }
    int fileId = fmp.lookupFileId(harness.getFileReference());
    TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId, btree.getRootPageId());
    TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("\n" + stats.toString());
    }
    TreeIndexBufferCacheWarmup bufferCacheWarmup = new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, fileId);
    bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 });
    btree.deactivate();
    btree.destroy();
    bufferCache.close();
}

Example 47 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class FieldPrefixNSMTest method createTuple.

private ITupleReference createTuple(IHyracksTaskContext ctx, int f0, int f1, int f2, boolean print) throws HyracksDataException {
    if (print) {
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("CREATING: " + f0 + " " + f1 + " " + f2);
        }
    }
    IFrame buf = new VSizeFrame(ctx);
    FrameTupleAppender appender = new FrameTupleAppender(buf);
    ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
    DataOutput dos = tb.getDataOutput();
    @SuppressWarnings("rawtypes") ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
    IFrameTupleAccessor accessor = new FrameTupleAccessor(recDesc);
    accessor.reset(buf.getBuffer());
    FrameTupleReference tuple = new FrameTupleReference();
    tb.reset();
    IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
    tb.addFieldEndOffset();
    IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
    tb.addFieldEndOffset();
    IntegerSerializerDeserializer.INSTANCE.serialize(f2, dos);
    tb.addFieldEndOffset();
    appender.reset(buf, true);
    appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
    tuple.reset(accessor, 0);
    return tuple;
}

Example 48 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class ExternalBTreeSearchOperatorNodePushable method open.

// We override the open function to search a specific version of the index
@Override
public void open() throws HyracksDataException {
    writer.open();
    accessor = new FrameTupleAccessor(inputRecDesc);
    indexHelper.open();
    index = indexHelper.getIndexInstance();
    if (retainMissing) {
        int fieldCount = getFieldCount();
        nonMatchTupleBuild = new ArrayTupleBuilder(fieldCount);
        DataOutput out = nonMatchTupleBuild.getDataOutput();
        for (int i = 0; i < fieldCount; i++) {
            try {
                nonMatchWriter.writeMissing(out);
            } catch (IOException e) {
                e.printStackTrace();
            }
            nonMatchTupleBuild.addFieldEndOffset();
        }
    } else {
        nonMatchTupleBuild = null;
    }
    ExternalBTreeWithBuddy externalIndex = (ExternalBTreeWithBuddy) index;
    try {
        searchPred = createSearchPredicate();
        tb = new ArrayTupleBuilder(recordDesc.getFieldCount());
        dos = tb.getDataOutput();
        appender = new FrameTupleAppender(new VSizeFrame(ctx));
        ISearchOperationCallback searchCallback = searchCallbackFactory.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
        // The next line is the reason we override this method
        indexAccessor = externalIndex.createAccessor(searchCallback, version);
        cursor = createCursor();
        if (retainInput) {
            frameTuple = new FrameTupleReference();
        }
    } catch (Throwable th) {
        throw new HyracksDataException(th);
    }
}

Example 49 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class FeedFrameUtil method getSampledFrame.

public static ByteBuffer getSampledFrame(IHyracksTaskContext ctx, FrameTupleAccessor fta, int sampleSize) throws HyracksDataException {
    NChooseKIterator it = new NChooseKIterator(fta.getTupleCount(), sampleSize);
    FrameTupleAppender appender = new FrameTupleAppender();
    IFrame sampledFrame = new VSizeFrame(ctx);
    appender.reset(sampledFrame, true);
    int nextTupleIndex = 0;
    while (it.hasNext()) {
        nextTupleIndex = it.next();
        appender.append(fta, nextTupleIndex);
    }
    return sampledFrame.getBuffer();
}

Example 50 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class FeedIntakeOperatorNodePushable method start.

@Override
protected void start() throws HyracksDataException, InterruptedException {
    try {
        writer.open();
        Thread.currentThread().setName("Intake Thread");
        FeedAdapter adapter = (FeedAdapter) adapterFactory.createAdapter(ctx, partition);
        adapterRuntimeManager = new AdapterRuntimeManager(ctx, runtimeId.getEntityId(), adapter, writer, partition);
        IFrame message = new VSizeFrame(ctx);
        TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
        /*
             * Set null feed message. Feed pipeline carries with it a message with each frame
             * Initially, the message is set to a null message that can be changed by feed adapters.
             * One use case is adapters which consume data sources that allow restartability. Such adapters
             * can propagate progress information through the ingestion pipeline to storage nodes
             */
        message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
        message.getBuffer().flip();
        adapterRuntimeManager.start();
        synchronized (adapterRuntimeManager) {
            while (!adapterRuntimeManager.isDone()) {
                adapterRuntimeManager.wait();
            }
        }
        if (adapterRuntimeManager.isFailed()) {
            throw new RuntimeDataException(ErrorCode.OPERATORS_FEED_INTAKE_OPERATOR_NODE_PUSHABLE_FAIL_AT_INGESTION);
        }
    } catch (Exception e) {
        /*
             * An Interrupted Exception is thrown if the Intake job cannot progress further due to failure of another
             * node involved in the Hyracks job. As the Intake job involves only the intake operator, the exception is
             * indicative of a failure at the sibling intake operator location. The surviving intake partitions must
             * continue to live and receive data from the external source.
             */
        writer.fail();
        throw e;
    } finally {
        writer.close();
    }
}