Search in sources :

Example 1 with LSMInsertDeleteOperatorNodePushable

use of org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable in project asterixdb by apache.

the class TestNodeController method getInsertPipeline.

public Pair<LSMInsertDeleteOperatorNodePushable, CommitRuntime> getInsertPipeline(IHyracksTaskContext ctx, Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields, int[] primaryKeyIndexes, List<Integer> primaryKeyIndicators, StorageComponentProvider storageComponentProvider) throws AlgebricksException, HyracksDataException {
    PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterFields, primaryKeyIndexes, primaryKeyIndicators, storageComponentProvider);
    IndexOperation op = IndexOperation.INSERT;
    IModificationOperationCallbackFactory modOpCallbackFactory = new PrimaryIndexModificationOperationCallbackFactory(getTxnJobId(), dataset.getDatasetId(), primaryIndexInfo.primaryKeyIndexes, TXN_SUBSYSTEM_PROVIDER, Operation.get(op), ResourceType.LSM_BTREE);
    IRecordDescriptorProvider recordDescProvider = primaryIndexInfo.getInsertRecordDescriptorProvider();
    IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), primaryIndexInfo.fileSplitProvider);
    LSMInsertDeleteOperatorNodePushable insertOp = new LSMInsertDeleteOperatorNodePushable(ctx, PARTITION, primaryIndexInfo.primaryIndexInsertFieldsPermutations, recordDescProvider.getInputRecordDescriptor(new ActivityId(new OperatorDescriptorId(0), 0), 0), op, true, indexHelperFactory, modOpCallbackFactory, null);
    CommitRuntime commitOp = new CommitRuntime(ctx, getTxnJobId(), dataset.getDatasetId(), primaryIndexInfo.primaryKeyIndexes, false, true, PARTITION, true);
    insertOp.setOutputFrameWriter(0, commitOp, primaryIndexInfo.rDesc);
    commitOp.setInputRecordDescriptor(0, primaryIndexInfo.rDesc);
    return Pair.of(insertOp, commitOp);
}
Also used : IndexOperation(org.apache.hyracks.storage.am.common.ophelpers.IndexOperation) PrimaryIndexModificationOperationCallbackFactory(org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory) LSMInsertDeleteOperatorNodePushable(org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable) OperatorDescriptorId(org.apache.hyracks.api.dataflow.OperatorDescriptorId) IIndexDataflowHelperFactory(org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory) ActivityId(org.apache.hyracks.api.dataflow.ActivityId) IRecordDescriptorProvider(org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider) IModificationOperationCallbackFactory(org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory) CommitRuntime(org.apache.asterix.transaction.management.runtime.CommitRuntime) IIndexDataflowHelperFactory(org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory) IndexDataflowHelperFactory(org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory)

Example 2 with LSMInsertDeleteOperatorNodePushable

use of org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable in project asterixdb by apache.

the class LogMarkerTest method testInsertWithSnapshot.

@Test
public void testInsertWithSnapshot() {
    try {
        TestNodeController nc = new TestNodeController(null, false);
        nc.init();
        StorageComponentProvider storageManager = new StorageComponentProvider();
        List<List<String>> partitioningKeys = new ArrayList<>();
        partitioningKeys.add(Collections.singletonList("key"));
        Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
        try {
            nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            IHyracksTaskContext ctx = nc.createTestContext(true);
            nc.newJobId();
            ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
            LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager).getLeft();
            insertOp.open();
            TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
            VSizeFrame frame = new VSizeFrame(ctx);
            VSizeFrame marker = new VSizeFrame(ctx);
            FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
            long markerId = 0L;
            for (int j = 0; j < NUM_OF_RECORDS; j++) {
                if (j % SNAPSHOT_SIZE == 0) {
                    marker.reset();
                    marker.getBuffer().put(MessagingFrameTupleAppender.MARKER_MESSAGE);
                    marker.getBuffer().putLong(markerId);
                    marker.getBuffer().flip();
                    markerId++;
                    TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx);
                    tupleAppender.flush(insertOp);
                }
                ITupleReference tuple = tupleGenerator.next();
                DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
            }
            if (tupleAppender.getTupleCount() > 0) {
                tupleAppender.write(insertOp, true);
            }
            insertOp.close();
            nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
            IIndexDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            dataflowHelper.open();
            LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance();
            LongPointable longPointable = LongPointable.FACTORY.createPointable();
            ComponentMetadataUtil.get(btree, ComponentMetadataUtil.MARKER_LSN_KEY, longPointable);
            long lsn = longPointable.getLong();
            int numOfMarkers = 0;
            LogReader logReader = (LogReader) nc.getTransactionSubsystem().getLogManager().getLogReader(false);
            long expectedMarkerId = markerId - 1;
            while (lsn >= 0) {
                numOfMarkers++;
                ILogRecord logRecord = logReader.read(lsn);
                lsn = logRecord.getPreviousMarkerLSN();
                long logMarkerId = logRecord.getMarker().getLong();
                Assert.assertEquals(expectedMarkerId, logMarkerId);
                expectedMarkerId--;
            }
            logReader.close();
            dataflowHelper.close();
            Assert.assertEquals(markerId, numOfMarkers);
            nc.newJobId();
            TestTupleCounterFrameWriter countOp = create(nc.getSearchOutputDesc(KEY_TYPES, RECORD_TYPE, META_TYPE), Collections.emptyList(), Collections.emptyList(), false);
            IPushRuntime emptyTupleOp = nc.getFullScanPipeline(countOp, ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager);
            emptyTupleOp.open();
            emptyTupleOp.close();
            Assert.assertEquals(NUM_OF_RECORDS, countOp.getCount());
        } finally {
            nc.deInit();
        }
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}
Also used : LSMInsertDeleteOperatorNodePushable(org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable) IIndexDataflowHelper(org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper) IPushRuntime(org.apache.hyracks.algebricks.runtime.base.IPushRuntime) ArrayList(java.util.ArrayList) TestTupleCounterFrameWriter(org.apache.asterix.app.data.gen.TestTupleCounterFrameWriter) NoMergePolicyFactory(org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory) MessagingFrameTupleAppender(org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender) FrameTupleAppender(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender) LongPointable(org.apache.hyracks.data.std.primitive.LongPointable) ArrayList(java.util.ArrayList) List(java.util.List) ILogRecord(org.apache.asterix.common.transactions.ILogRecord) Dataset(org.apache.asterix.metadata.entities.Dataset) InternalDatasetDetails(org.apache.asterix.metadata.entities.InternalDatasetDetails) ITransactionContext(org.apache.asterix.common.transactions.ITransactionContext) TupleGenerator(org.apache.asterix.app.data.gen.TupleGenerator) StorageComponentProvider(org.apache.asterix.file.StorageComponentProvider) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame) IHyracksTaskContext(org.apache.hyracks.api.context.IHyracksTaskContext) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) TestNodeController(org.apache.asterix.app.bootstrap.TestNodeController) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) LogReader(org.apache.asterix.transaction.management.service.logging.LogReader) Test(org.junit.Test)

Example 3 with LSMInsertDeleteOperatorNodePushable

use of org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable in project asterixdb by apache.

the class CheckpointingTest method testDeleteOldLogFiles.

@Test
public void testDeleteOldLogFiles() {
    try {
        TestNodeController nc = new TestNodeController(new File(TEST_CONFIG_FILE_PATH).getAbsolutePath(), false);
        StorageComponentProvider storageManager = new StorageComponentProvider();
        nc.init();
        List<List<String>> partitioningKeys = new ArrayList<>();
        partitioningKeys.add(Collections.singletonList("key"));
        Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
        try {
            nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATOR_LIST);
            IHyracksTaskContext ctx = nc.createTestContext(false);
            nc.newJobId();
            ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
            // Prepare insert operation
            LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATOR_LIST, storageManager).getLeft();
            insertOp.open();
            TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATOR, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
            VSizeFrame frame = new VSizeFrame(ctx);
            FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
            IRecoveryManager recoveryManager = nc.getTransactionSubsystem().getRecoveryManager();
            ICheckpointManager checkpointManager = nc.getTransactionSubsystem().getCheckpointManager();
            LogManager logManager = (LogManager) nc.getTransactionSubsystem().getLogManager();
            // Number of log files after node startup should be one
            int numberOfLogFiles = logManager.getLogFileIds().size();
            Assert.assertEquals(1, numberOfLogFiles);
            // Low-water mark LSN
            long lowWaterMarkLSN = recoveryManager.getMinFirstLSN();
            // Low-water mark log file id
            long initialLowWaterMarkFileId = logManager.getLogFileId(lowWaterMarkLSN);
            // Initial Low-water mark should be in the only available log file
            Assert.assertEquals(initialLowWaterMarkFileId, logManager.getLogFileIds().get(0).longValue());
            // Insert records until a new log file is created
            while (logManager.getLogFileIds().size() == 1) {
                ITupleReference tuple = tupleGenerator.next();
                DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
            }
            // Check if the new low-water mark is still in the initial low-water mark log file
            lowWaterMarkLSN = recoveryManager.getMinFirstLSN();
            long currentLowWaterMarkLogFileId = logManager.getLogFileId(lowWaterMarkLSN);
            if (currentLowWaterMarkLogFileId == initialLowWaterMarkFileId) {
                /*
                     * Make sure checkpoint will not delete the initial log file since
                     * the low-water mark is still in it (i.e. it is still required for
                     * recovery)
                     */
                int numberOfLogFilesBeforeCheckpoint = logManager.getLogFileIds().size();
                checkpointManager.tryCheckpoint(logManager.getAppendLSN());
                int numberOfLogFilesAfterCheckpoint = logManager.getLogFileIds().size();
                Assert.assertEquals(numberOfLogFilesBeforeCheckpoint, numberOfLogFilesAfterCheckpoint);
                /*
                     * Insert records until the low-water mark is not in the initialLowWaterMarkFileId
                     * either because of the asynchronous flush caused by the previous checkpoint or a flush
                     * due to the dataset memory budget getting full.
                     */
                while (currentLowWaterMarkLogFileId == initialLowWaterMarkFileId) {
                    ITupleReference tuple = tupleGenerator.next();
                    DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
                    lowWaterMarkLSN = recoveryManager.getMinFirstLSN();
                    currentLowWaterMarkLogFileId = logManager.getLogFileId(lowWaterMarkLSN);
                }
            }
            /*
                 * At this point, the low-water mark is not in the initialLowWaterMarkFileId, so
                 * a checkpoint should delete it.
                 */
            checkpointManager.tryCheckpoint(recoveryManager.getMinFirstLSN());
            // Validate initialLowWaterMarkFileId was deleted
            for (Long fileId : logManager.getLogFileIds()) {
                Assert.assertNotEquals(initialLowWaterMarkFileId, fileId.longValue());
            }
            if (tupleAppender.getTupleCount() > 0) {
                tupleAppender.write(insertOp, true);
            }
            insertOp.close();
            nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
        } finally {
            nc.deInit();
        }
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}
Also used : LSMInsertDeleteOperatorNodePushable(org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable) ICheckpointManager(org.apache.asterix.common.transactions.ICheckpointManager) Dataset(org.apache.asterix.metadata.entities.Dataset) InternalDatasetDetails(org.apache.asterix.metadata.entities.InternalDatasetDetails) ITransactionContext(org.apache.asterix.common.transactions.ITransactionContext) ArrayList(java.util.ArrayList) TupleGenerator(org.apache.asterix.app.data.gen.TupleGenerator) StorageComponentProvider(org.apache.asterix.file.StorageComponentProvider) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame) NoMergePolicyFactory(org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory) IRecoveryManager(org.apache.asterix.common.transactions.IRecoveryManager) IHyracksTaskContext(org.apache.hyracks.api.context.IHyracksTaskContext) TestNodeController(org.apache.asterix.app.bootstrap.TestNodeController) FrameTupleAppender(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) ArrayList(java.util.ArrayList) List(java.util.List) File(java.io.File) LogManager(org.apache.asterix.transaction.management.service.logging.LogManager) Test(org.junit.Test)

Example 4 with LSMInsertDeleteOperatorNodePushable

use of org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable in project asterixdb by apache.

the class FeedMetaStoreNodePushable method initializeNewFeedRuntime.

private void initializeNewFeedRuntime(ActiveRuntimeId runtimeId) throws Exception {
    fta = new FrameTupleAccessor(recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0));
    insertOperator.setOutputFrameWriter(0, writer, recordDesc);
    if (insertOperator instanceof LSMInsertDeleteOperatorNodePushable) {
        LSMInsertDeleteOperatorNodePushable indexOp = (LSMInsertDeleteOperatorNodePushable) insertOperator;
        if (!indexOp.isPrimary()) {
            writer = insertOperator;
            return;
        }
    }
    if (policyAccessor.flowControlEnabled()) {
        writer = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, insertOperator, policyAccessor, fta, feedManager.getFramePool());
    } else {
        writer = new SyncFeedRuntimeInputHandler(ctx, insertOperator, fta);
    }
}
Also used : LSMInsertDeleteOperatorNodePushable(org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable) SyncFeedRuntimeInputHandler(org.apache.asterix.external.feed.dataflow.SyncFeedRuntimeInputHandler) FeedRuntimeInputHandler(org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler) SyncFeedRuntimeInputHandler(org.apache.asterix.external.feed.dataflow.SyncFeedRuntimeInputHandler) FrameTupleAccessor(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor)

Aggregations

LSMInsertDeleteOperatorNodePushable (org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable)4 ArrayList (java.util.ArrayList)2 List (java.util.List)2 TestNodeController (org.apache.asterix.app.bootstrap.TestNodeController)2 TupleGenerator (org.apache.asterix.app.data.gen.TupleGenerator)2 ITransactionContext (org.apache.asterix.common.transactions.ITransactionContext)2 StorageComponentProvider (org.apache.asterix.file.StorageComponentProvider)2 Dataset (org.apache.asterix.metadata.entities.Dataset)2 InternalDatasetDetails (org.apache.asterix.metadata.entities.InternalDatasetDetails)2 VSizeFrame (org.apache.hyracks.api.comm.VSizeFrame)2 IHyracksTaskContext (org.apache.hyracks.api.context.IHyracksTaskContext)2 FrameTupleAppender (org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender)2 ITupleReference (org.apache.hyracks.dataflow.common.data.accessors.ITupleReference)2 NoMergePolicyFactory (org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory)2 Test (org.junit.Test)2 File (java.io.File)1 TestTupleCounterFrameWriter (org.apache.asterix.app.data.gen.TestTupleCounterFrameWriter)1 ICheckpointManager (org.apache.asterix.common.transactions.ICheckpointManager)1 ILogRecord (org.apache.asterix.common.transactions.ILogRecord)1 IRecoveryManager (org.apache.asterix.common.transactions.IRecoveryManager)1