use of org.apache.asterix.common.transactions.ITransactionContext in project asterixdb by apache.
the class MetadataNode method commitTransaction.
@Override
public void commitTransaction(JobId jobId) throws RemoteException, ACIDException {
ITransactionContext txnCtx = transactionSubsystem.getTransactionManager().getTransactionContext(jobId, false);
transactionSubsystem.getTransactionManager().commitTransaction(txnCtx, DatasetId.NULL, -1);
}
use of org.apache.asterix.common.transactions.ITransactionContext in project asterixdb by apache.
the class MetadataNode method unlock.
@Override
public void unlock(JobId jobId, byte lockMode) throws ACIDException, RemoteException {
ITransactionContext txnCtx = transactionSubsystem.getTransactionManager().getTransactionContext(jobId, false);
transactionSubsystem.getLockManager().unlock(METADATA_DATASET_ID, -1, lockMode, txnCtx);
}
use of org.apache.asterix.common.transactions.ITransactionContext in project asterixdb by apache.
the class LogMarkerTest method testInsertWithSnapshot.
@Test
public void testInsertWithSnapshot() {
try {
TestNodeController nc = new TestNodeController(null, false);
nc.init();
StorageComponentProvider storageManager = new StorageComponentProvider();
List<List<String>> partitioningKeys = new ArrayList<>();
partitioningKeys.add(Collections.singletonList("key"));
Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
try {
nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
IHyracksTaskContext ctx = nc.createTestContext(true);
nc.newJobId();
ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager).getLeft();
insertOp.open();
TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
VSizeFrame frame = new VSizeFrame(ctx);
VSizeFrame marker = new VSizeFrame(ctx);
FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
long markerId = 0L;
for (int j = 0; j < NUM_OF_RECORDS; j++) {
if (j % SNAPSHOT_SIZE == 0) {
marker.reset();
marker.getBuffer().put(MessagingFrameTupleAppender.MARKER_MESSAGE);
marker.getBuffer().putLong(markerId);
marker.getBuffer().flip();
markerId++;
TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx);
tupleAppender.flush(insertOp);
}
ITupleReference tuple = tupleGenerator.next();
DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
}
if (tupleAppender.getTupleCount() > 0) {
tupleAppender.write(insertOp, true);
}
insertOp.close();
nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
IIndexDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
dataflowHelper.open();
LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance();
LongPointable longPointable = LongPointable.FACTORY.createPointable();
ComponentMetadataUtil.get(btree, ComponentMetadataUtil.MARKER_LSN_KEY, longPointable);
long lsn = longPointable.getLong();
int numOfMarkers = 0;
LogReader logReader = (LogReader) nc.getTransactionSubsystem().getLogManager().getLogReader(false);
long expectedMarkerId = markerId - 1;
while (lsn >= 0) {
numOfMarkers++;
ILogRecord logRecord = logReader.read(lsn);
lsn = logRecord.getPreviousMarkerLSN();
long logMarkerId = logRecord.getMarker().getLong();
Assert.assertEquals(expectedMarkerId, logMarkerId);
expectedMarkerId--;
}
logReader.close();
dataflowHelper.close();
Assert.assertEquals(markerId, numOfMarkers);
nc.newJobId();
TestTupleCounterFrameWriter countOp = create(nc.getSearchOutputDesc(KEY_TYPES, RECORD_TYPE, META_TYPE), Collections.emptyList(), Collections.emptyList(), false);
IPushRuntime emptyTupleOp = nc.getFullScanPipeline(countOp, ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager);
emptyTupleOp.open();
emptyTupleOp.close();
Assert.assertEquals(NUM_OF_RECORDS, countOp.getCount());
} finally {
nc.deInit();
}
} catch (Throwable e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
use of org.apache.asterix.common.transactions.ITransactionContext in project asterixdb by apache.
the class JobEventListenerFactory method createListener.
@Override
public IJobletEventListener createListener(final IHyracksJobletContext jobletContext) {
return new IJobletEventListener() {
@Override
public void jobletFinish(JobStatus jobStatus) {
try {
ITransactionManager txnManager = ((INcApplicationContext) jobletContext.getServiceContext().getApplicationContext()).getTransactionSubsystem().getTransactionManager();
ITransactionContext txnContext = txnManager.getTransactionContext(jobId, false);
txnContext.setWriteTxn(transactionalWrite);
txnManager.completedTransaction(txnContext, DatasetId.NULL, -1, !(jobStatus == JobStatus.FAILURE));
} catch (ACIDException e) {
throw new Error(e);
}
}
@Override
public void jobletStart() {
try {
((INcApplicationContext) jobletContext.getServiceContext().getApplicationContext()).getTransactionSubsystem().getTransactionManager().getTransactionContext(jobId, true);
} catch (ACIDException e) {
throw new Error(e);
}
}
};
}
use of org.apache.asterix.common.transactions.ITransactionContext in project asterixdb by apache.
the class CheckpointingTest method testDeleteOldLogFiles.
@Test
public void testDeleteOldLogFiles() {
try {
TestNodeController nc = new TestNodeController(new File(TEST_CONFIG_FILE_PATH).getAbsolutePath(), false);
StorageComponentProvider storageManager = new StorageComponentProvider();
nc.init();
List<List<String>> partitioningKeys = new ArrayList<>();
partitioningKeys.add(Collections.singletonList("key"));
Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
try {
nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATOR_LIST);
IHyracksTaskContext ctx = nc.createTestContext(false);
nc.newJobId();
ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
// Prepare insert operation
LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATOR_LIST, storageManager).getLeft();
insertOp.open();
TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATOR, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
VSizeFrame frame = new VSizeFrame(ctx);
FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
IRecoveryManager recoveryManager = nc.getTransactionSubsystem().getRecoveryManager();
ICheckpointManager checkpointManager = nc.getTransactionSubsystem().getCheckpointManager();
LogManager logManager = (LogManager) nc.getTransactionSubsystem().getLogManager();
// Number of log files after node startup should be one
int numberOfLogFiles = logManager.getLogFileIds().size();
Assert.assertEquals(1, numberOfLogFiles);
// Low-water mark LSN
long lowWaterMarkLSN = recoveryManager.getMinFirstLSN();
// Low-water mark log file id
long initialLowWaterMarkFileId = logManager.getLogFileId(lowWaterMarkLSN);
// Initial Low-water mark should be in the only available log file
Assert.assertEquals(initialLowWaterMarkFileId, logManager.getLogFileIds().get(0).longValue());
// Insert records until a new log file is created
while (logManager.getLogFileIds().size() == 1) {
ITupleReference tuple = tupleGenerator.next();
DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
}
// Check if the new low-water mark is still in the initial low-water mark log file
lowWaterMarkLSN = recoveryManager.getMinFirstLSN();
long currentLowWaterMarkLogFileId = logManager.getLogFileId(lowWaterMarkLSN);
if (currentLowWaterMarkLogFileId == initialLowWaterMarkFileId) {
/*
* Make sure checkpoint will not delete the initial log file since
* the low-water mark is still in it (i.e. it is still required for
* recovery)
*/
int numberOfLogFilesBeforeCheckpoint = logManager.getLogFileIds().size();
checkpointManager.tryCheckpoint(logManager.getAppendLSN());
int numberOfLogFilesAfterCheckpoint = logManager.getLogFileIds().size();
Assert.assertEquals(numberOfLogFilesBeforeCheckpoint, numberOfLogFilesAfterCheckpoint);
/*
* Insert records until the low-water mark is not in the initialLowWaterMarkFileId
* either because of the asynchronous flush caused by the previous checkpoint or a flush
* due to the dataset memory budget getting full.
*/
while (currentLowWaterMarkLogFileId == initialLowWaterMarkFileId) {
ITupleReference tuple = tupleGenerator.next();
DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
lowWaterMarkLSN = recoveryManager.getMinFirstLSN();
currentLowWaterMarkLogFileId = logManager.getLogFileId(lowWaterMarkLSN);
}
}
/*
* At this point, the low-water mark is not in the initialLowWaterMarkFileId, so
* a checkpoint should delete it.
*/
checkpointManager.tryCheckpoint(recoveryManager.getMinFirstLSN());
// Validate initialLowWaterMarkFileId was deleted
for (Long fileId : logManager.getLogFileIds()) {
Assert.assertNotEquals(initialLowWaterMarkFileId, fileId.longValue());
}
if (tupleAppender.getTupleCount() > 0) {
tupleAppender.write(insertOp, true);
}
insertOp.close();
nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
} finally {
nc.deInit();
}
} catch (Throwable e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
Aggregations