Examples with ByteBufferInputStream org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream used on opensource projects

Example 6 with ByteBufferInputStream

use of org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream in project asterixdb by apache.

the class AbstractRunGeneratorTest method assertFTADataIsSorted.

static int assertFTADataIsSorted(IFrameTupleAccessor fta, Map<Integer, String> keyValuePair, int preKey) throws HyracksDataException {
    ByteBufferInputStream bbis = new ByteBufferInputStream();
    DataInputStream di = new DataInputStream(bbis);
    for (int i = 0; i < fta.getTupleCount(); i++) {
        bbis.setByteBuffer(fta.getBuffer(), fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 0) + fta.getFieldSlotsLength());
        int key = (int) RecordDesc.getFields()[0].deserialize(di);
        bbis.setByteBuffer(fta.getBuffer(), fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 1) + fta.getFieldSlotsLength());
        String value = (String) RecordDesc.getFields()[1].deserialize(di);
        if (!keyValuePair.get(key).equals(value)) {
            assertTrue(false);
        }
        keyValuePair.remove(key);
        assertTrue(key >= preKey);
        preKey = key;
    }
    return preKey;
}

Example 7 with ByteBufferInputStream

use of org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream in project asterixdb by apache.

the class RunMergingFrameReaderTest method assertFrameIsSorted.

private void assertFrameIsSorted(IFrame frame, List<Map<Integer, String>> keyValueMapList) throws HyracksDataException {
    FrameTupleAccessor fta = new FrameTupleAccessor(RecordDesc);
    ByteBufferInputStream bbis = new ByteBufferInputStream();
    DataInputStream di = new DataInputStream(bbis);
    fta.reset(frame.getBuffer());
    //        fta.prettyPrint();
    int preKey = Integer.MIN_VALUE;
    for (int i = 0; i < fta.getTupleCount(); i++) {
        bbis.setByteBuffer(fta.getBuffer(), fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 0) + fta.getFieldSlotsLength());
        int key = (int) RecordDesc.getFields()[0].deserialize(di);
        bbis.setByteBuffer(fta.getBuffer(), fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 1) + fta.getFieldSlotsLength());
        String value = (String) RecordDesc.getFields()[1].deserialize(di);
        boolean found = false;
        for (Map<Integer, String> map : keyValueMapList) {
            if (map.containsKey(key) && map.get(key).equals(value)) {
                found = true;
                map.remove(key);
                break;
            }
        }
        assertTrue(found);
        assertTrue(preKey <= key);
        preKey = key;
    }
}

Example 8 with ByteBufferInputStream

use of org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream in project asterixdb by apache.

the class PlainFileWriterOperatorDescriptor method createPushRuntime.

/*
     * (non-Javadoc)
     *
     * @see
     * org.apache.hyracks.api.dataflow.IActivityNode#createPushRuntime(edu.
     * uci.ics.hyracks.api.context.IHyracksContext,
     * org.apache.hyracks.api.job.IOperatorEnvironment,
     * org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider, int,
     * int)
     */
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) throws HyracksDataException {
    // Output files
    final FileSplit[] splits = fileSplitProvider.getFileSplits();
    IIOManager ioManager = ctx.getIoManager();
    // Frame accessor
    final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
    // Record descriptor
    final RecordDescriptor recordDescriptor = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0);
    return new AbstractUnaryInputSinkOperatorNodePushable() {

        private BufferedWriter out;

        private ByteBufferInputStream bbis;

        private DataInputStream di;

        @Override
        public void open() throws HyracksDataException {
            try {
                out = new BufferedWriter(new FileWriter(splits[partition].getFile(ioManager)));
                bbis = new ByteBufferInputStream();
                di = new DataInputStream(bbis);
            } catch (Exception e) {
                throw new HyracksDataException(e);
            }
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            try {
                frameTupleAccessor.reset(buffer);
                for (int tIndex = 0; tIndex < frameTupleAccessor.getTupleCount(); tIndex++) {
                    int start = frameTupleAccessor.getTupleStartOffset(tIndex) + frameTupleAccessor.getFieldSlotsLength();
                    bbis.setByteBuffer(buffer, start);
                    Object[] record = new Object[recordDescriptor.getFieldCount()];
                    for (int i = 0; i < record.length; ++i) {
                        Object instance = recordDescriptor.getFields()[i].deserialize(di);
                        if (i == 0) {
                            out.write(String.valueOf(instance));
                        } else {
                            out.write(delim + String.valueOf(instance));
                        }
                    }
                    out.write("\n");
                }
            } catch (IOException ex) {
                throw new HyracksDataException(ex);
            }
        }

        @Override
        public void fail() throws HyracksDataException {
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                out.close();
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

Example 9 with ByteBufferInputStream

use of org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream in project asterixdb by apache.

the class ResultSerializerFactoryProvider method getResultSerializerFactoryProvider.

public IResultSerializerFactory getResultSerializerFactoryProvider() {
    return new IResultSerializerFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IResultSerializer createResultSerializer(final RecordDescriptor recordDesc, final PrintStream printStream) {
            return new IResultSerializer() {

                private static final long serialVersionUID = 1L;

                ByteBufferInputStream bbis = new ByteBufferInputStream();

                DataInputStream di = new DataInputStream(bbis);

                @Override
                public void init() throws HyracksDataException {
                }

                @Override
                public boolean appendTuple(IFrameTupleAccessor tAccess, int tIdx) throws HyracksDataException {
                    int start = tAccess.getTupleStartOffset(tIdx) + tAccess.getFieldSlotsLength();
                    bbis.setByteBuffer(tAccess.getBuffer(), start);
                    Object[] record = new Object[recordDesc.getFieldCount()];
                    for (int i = 0; i < record.length; ++i) {
                        Object instance = recordDesc.getFields()[i].deserialize(di);
                        if (i == 0) {
                            printStream.print(String.valueOf(instance));
                        } else {
                            printStream.print(", " + String.valueOf(instance));
                        }
                    }
                    printStream.println();
                    return true;
                }
            };
        }
    };
}

Example 10 with ByteBufferInputStream

use of org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream in project asterixdb by apache.

the class AbstractMultiNCIntegrationTest method runTest.

protected void runTest(JobSpecification spec) throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(spec.toJSON().asText());
    }
    JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(jobId.toString());
    }
    int nReaders = 1;
    FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize());
    VSizeFrame resultFrame = new VSizeFrame(resultDisplayFrameMgr);
    IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
    if (!spec.getResultSetIds().isEmpty()) {
        IHyracksDataset hyracksDataset = new HyracksDataset(hcc, spec.getFrameSize(), nReaders);
        IHyracksDatasetReader reader = hyracksDataset.createReader(jobId, spec.getResultSetIds().get(0));
        ObjectMapper om = new ObjectMapper();
        ArrayNode resultRecords = om.createArrayNode();
        ByteBufferInputStream bbis = new ByteBufferInputStream();
        int readSize = reader.read(resultFrame);
        while (readSize > 0) {
            try {
                frameTupleAccessor.reset(resultFrame.getBuffer());
                for (int tIndex = 0; tIndex < frameTupleAccessor.getTupleCount(); tIndex++) {
                    int start = frameTupleAccessor.getTupleStartOffset(tIndex);
                    int length = frameTupleAccessor.getTupleEndOffset(tIndex) - start;
                    bbis.setByteBuffer(resultFrame.getBuffer(), start);
                    byte[] recordBytes = new byte[length];
                    bbis.read(recordBytes, 0, length);
                    resultRecords.add(new String(recordBytes, 0, length));
                }
            } finally {
                try {
                    bbis.close();
                } catch (IOException e) {
                    throw new HyracksDataException(e);
                }
            }
            readSize = reader.read(resultFrame);
        }
    }
    hcc.waitForCompletion(jobId);
    dumpOutputFiles();
}