Search in sources :

Example 1 with FileIndexTupleTranslator

use of org.apache.asterix.external.indexing.FileIndexTupleTranslator in project asterixdb by apache.

the class ExternalFilesIndexModificationOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    return new AbstractOperatorNodePushable() {

        @Override
        public void initialize() throws HyracksDataException {
            final IIndexDataflowHelper indexHelper = dataflowHelperFactory.create(ctx, partition);
            FileIndexTupleTranslator filesTupleTranslator = new FileIndexTupleTranslator();
            // Open and get
            indexHelper.open();
            IIndex index = indexHelper.getIndexInstance();
            LSMTwoPCBTreeBulkLoader bulkLoader = null;
            try {
                bulkLoader = (LSMTwoPCBTreeBulkLoader) ((ExternalBTree) index).createTransactionBulkLoader(BTree.DEFAULT_FILL_FACTOR, false, files.size());
                // The files must be ordered according to their numbers
                for (ExternalFile file : files) {
                    switch(file.getPendingOp()) {
                        case ADD_OP:
                        case APPEND_OP:
                            bulkLoader.add(filesTupleTranslator.getTupleFromFile(file));
                            break;
                        case DROP_OP:
                            bulkLoader.delete(filesTupleTranslator.getTupleFromFile(file));
                            break;
                        case NO_OP:
                            break;
                        default:
                            throw HyracksDataException.create(ErrorCode.UNKNOWN_EXTERNAL_FILE_PENDING_OP, file.getPendingOp());
                    }
                }
                bulkLoader.end();
            } catch (Exception e) {
                if (bulkLoader != null) {
                    bulkLoader.abort();
                }
                throw HyracksDataException.create(e);
            } finally {
                indexHelper.close();
            }
        }

        @Override
        public void deinitialize() throws HyracksDataException {
        }

        @Override
        public int getInputArity() {
            return 0;
        }

        @Override
        public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) throws HyracksDataException {
        }

        @Override
        public IFrameWriter getInputFrameWriter(int index) {
            return null;
        }
    };
}
Also used : IIndex(org.apache.hyracks.storage.common.IIndex) IFrameWriter(org.apache.hyracks.api.comm.IFrameWriter) FileIndexTupleTranslator(org.apache.asterix.external.indexing.FileIndexTupleTranslator) IIndexDataflowHelper(org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper) AbstractOperatorNodePushable(org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) ExternalFile(org.apache.asterix.external.indexing.ExternalFile) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) LSMTwoPCBTreeBulkLoader(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree.LSMTwoPCBTreeBulkLoader)

Example 2 with FileIndexTupleTranslator

use of org.apache.asterix.external.indexing.FileIndexTupleTranslator in project asterixdb by apache.

the class ExternalFilesIndexCreateOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    return new AbstractOperatorNodePushable() {

        @Override
        public void initialize() throws HyracksDataException {
            IIndexBuilder indexBuilder = indexBuilderFactory.create(ctx, partition);
            IIndexDataflowHelper indexHelper = dataflowHelperFactory.create(ctx, partition);
            FileIndexTupleTranslator filesTupleTranslator = new FileIndexTupleTranslator();
            // Build the index
            indexBuilder.build();
            // Open the index
            indexHelper.open();
            try {
                IIndex index = indexHelper.getIndexInstance();
                // Create bulk loader
                IIndexBulkLoader bulkLoader = index.createBulkLoader(BTree.DEFAULT_FILL_FACTOR, false, files.size(), false);
                // Load files
                for (ExternalFile file : files) {
                    bulkLoader.add(filesTupleTranslator.getTupleFromFile(file));
                }
                bulkLoader.end();
            } finally {
                indexHelper.close();
            }
        }

        @Override
        public void deinitialize() throws HyracksDataException {
        }

        @Override
        public int getInputArity() {
            return 0;
        }

        @Override
        public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) throws HyracksDataException {
        }

        @Override
        public IFrameWriter getInputFrameWriter(int index) {
            return null;
        }
    };
}
Also used : IIndex(org.apache.hyracks.storage.common.IIndex) IIndexBuilder(org.apache.hyracks.storage.am.common.api.IIndexBuilder) IFrameWriter(org.apache.hyracks.api.comm.IFrameWriter) FileIndexTupleTranslator(org.apache.asterix.external.indexing.FileIndexTupleTranslator) IIndexDataflowHelper(org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper) AbstractOperatorNodePushable(org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable) IIndexBulkLoader(org.apache.hyracks.storage.common.IIndexBulkLoader) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) ExternalFile(org.apache.asterix.external.indexing.ExternalFile)

Aggregations

ExternalFile (org.apache.asterix.external.indexing.ExternalFile)2 FileIndexTupleTranslator (org.apache.asterix.external.indexing.FileIndexTupleTranslator)2 IFrameWriter (org.apache.hyracks.api.comm.IFrameWriter)2 RecordDescriptor (org.apache.hyracks.api.dataflow.value.RecordDescriptor)2 AbstractOperatorNodePushable (org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable)2 IIndexDataflowHelper (org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper)2 IIndex (org.apache.hyracks.storage.common.IIndex)2 HyracksDataException (org.apache.hyracks.api.exceptions.HyracksDataException)1 IIndexBuilder (org.apache.hyracks.storage.am.common.api.IIndexBuilder)1 ExternalBTree (org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree)1 LSMTwoPCBTreeBulkLoader (org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree.LSMTwoPCBTreeBulkLoader)1 IIndexBulkLoader (org.apache.hyracks.storage.common.IIndexBulkLoader)1