use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.
the class TreeIndexTestUtils method deleteTuples.
@SuppressWarnings("unchecked")
public void deleteTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
ArrayTupleBuilder deleteTupleBuilder = createDeleteTupleBuilder(ctx);
ArrayTupleReference deleteTuple = new ArrayTupleReference();
int numCheckTuples = ctx.getCheckTuples().size();
// Copy CheckTuple references into array, so we can randomly pick from
// there.
CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
int idx = 0;
Iterator<CheckTuple> iter = ctx.getCheckTuples().iterator();
while (iter.hasNext()) {
CheckTuple checkTuple = iter.next();
checkTuples[idx++] = checkTuple;
}
for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
if (LOGGER.isLoggable(Level.INFO)) {
if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
LOGGER.info("Deleting Tuple " + (i + 1) + "/" + numTuples);
}
}
int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
CheckTuple checkTuple = checkTuples[checkTupleIdx];
createTupleFromCheckTuple(checkTuple, deleteTupleBuilder, deleteTuple, ctx.getFieldSerdes());
ctx.getIndexAccessor().delete(deleteTuple);
// Remove check tuple from expected results.
ctx.deleteCheckTuple(checkTuple, ctx.getCheckTuples());
// Swap with last "valid" CheckTuple.
CheckTuple tmp = checkTuples[numCheckTuples - 1];
checkTuples[numCheckTuples - 1] = checkTuple;
checkTuples[checkTupleIdx] = tmp;
numCheckTuples--;
}
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.
the class TreeIndexTestUtils method addFilterField.
protected void addFilterField(IIndexTestContext ctx, MutablePair<ITupleReference, ITupleReference> minMax) throws HyracksDataException {
//Duplicate the PK field as a filter field at the end of the tuple to be inserted.
int filterField = ctx.getFieldCount();
ITupleReference currTuple = ctx.getTuple();
ArrayTupleBuilder filterBuilder = new ArrayTupleBuilder(1);
filterBuilder.addField(currTuple.getFieldData(filterField), currTuple.getFieldStart(filterField), currTuple.getFieldLength(filterField));
IBinaryComparator comparator = ctx.getComparatorFactories()[0].createBinaryComparator();
ArrayTupleReference filterOnlyTuple = new ArrayTupleReference();
filterOnlyTuple.reset(filterBuilder.getFieldEndOffsets(), filterBuilder.getByteArray());
if (minMax == null) {
minMax = MutablePair.of(filterOnlyTuple, filterOnlyTuple);
} else if (compareFilterTuples(minMax.getLeft(), filterOnlyTuple, comparator) > 0) {
minMax.setLeft(filterOnlyTuple);
} else if (compareFilterTuples(minMax.getRight(), filterOnlyTuple, comparator) < 0) {
minMax.setRight(filterOnlyTuple);
}
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.
the class OnDiskInvertedIndex method validate.
@Override
public void validate() throws HyracksDataException {
btree.validate();
// Scan the btree and validate the order of elements in each inverted-list.
IIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
int[] fieldPermutation = new int[tokenTypeTraits.length];
for (int i = 0; i < tokenTypeTraits.length; i++) {
fieldPermutation[i] = i;
}
PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
try {
// Search key for finding an inverted-list in the actual index.
ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
ArrayTupleReference prevTuple = new ArrayTupleReference();
btreeAccessor.search(btreeCursor, rangePred);
while (btreeCursor.hasNext()) {
btreeCursor.next();
tokenTuple.reset(btreeCursor.getTuple());
// Validate inverted list by checking that the elements are totally ordered.
invIndexAccessor.openInvertedListCursor(invListCursor, tokenTuple);
invListCursor.pinPages();
try {
if (invListCursor.hasNext()) {
invListCursor.next();
ITupleReference invListElement = invListCursor.getTuple();
// Initialize prev tuple.
TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
}
while (invListCursor.hasNext()) {
invListCursor.next();
ITupleReference invListElement = invListCursor.getTuple();
// Compare with previous element.
if (invListCmp.compare(invListElement, prevTuple) <= 0) {
throw new HyracksDataException("Index validation failed.");
}
// Set new prevTuple.
TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
}
} finally {
invListCursor.unpinPages();
}
}
} finally {
btreeCursor.close();
}
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.
the class LSMInvertedIndexTestUtils method bulkLoadInvIndex.
public static void bulkLoadInvIndex(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen, int numDocs, boolean appendOnly) throws HyracksDataException, IOException {
SortedSet<CheckTuple> tmpMemIndex = new TreeSet<>();
// First generate the expected index by inserting the documents one-by-one.
for (int i = 0; i < numDocs; i++) {
ITupleReference tuple = tupleGen.next();
testCtx.insertCheckTuples(tuple, tmpMemIndex);
}
ISerializerDeserializer[] fieldSerdes = testCtx.getFieldSerdes();
// Use the expected index to bulk-load the actual index.
IIndexBulkLoader bulkLoader = testCtx.getIndex().createBulkLoader(1.0f, false, numDocs, true);
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(testCtx.getFieldSerdes().length);
ArrayTupleReference tuple = new ArrayTupleReference();
Iterator<CheckTuple> checkTupleIter = tmpMemIndex.iterator();
while (checkTupleIter.hasNext()) {
CheckTuple checkTuple = checkTupleIter.next();
OrderedIndexTestUtils.createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, fieldSerdes);
bulkLoader.add(tuple);
}
bulkLoader.end();
// Add all check tuples from the temp index to the text context.
testCtx.getCheckTuples().addAll(tmpMemIndex);
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.
the class LSMInvertedIndexTestUtils method compareActualAndExpectedIndexes.
/**
* Compares actual and expected indexes by comparing their inverted-lists one by one. Exercises the openInvertedListCursor() method of the inverted-index accessor.
*/
@SuppressWarnings("unchecked")
public static void compareActualAndExpectedIndexes(LSMInvertedIndexTestContext testCtx) throws HyracksDataException {
IInvertedIndex invIndex = (IInvertedIndex) testCtx.getIndex();
ISerializerDeserializer[] fieldSerdes = testCtx.getFieldSerdes();
MultiComparator invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories());
IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) testCtx.getIndexAccessor();
int tokenFieldCount = invIndex.getTokenTypeTraits().length;
int invListFieldCount = invIndex.getInvListTypeTraits().length;
// All tokens that were inserted into the indexes.
Iterator<Comparable> tokensIter = testCtx.getAllTokens().iterator();
// Search key for finding an inverted-list in the actual index.
ArrayTupleBuilder searchKeyBuilder = new ArrayTupleBuilder(tokenFieldCount);
ArrayTupleReference searchKey = new ArrayTupleReference();
// Cursor over inverted list from actual index.
IInvertedListCursor actualInvListCursor = invIndexAccessor.createInvertedListCursor();
// Helpers for generating a serialized inverted-list element from a CheckTuple from the expected index.
ArrayTupleBuilder expectedBuilder = new ArrayTupleBuilder(fieldSerdes.length);
// Includes the token fields.
ArrayTupleReference completeExpectedTuple = new ArrayTupleReference();
// Field permutation and permuting tuple reference to strip away token fields from completeExpectedTuple.
int[] fieldPermutation = new int[invListFieldCount];
for (int i = 0; i < fieldPermutation.length; i++) {
fieldPermutation[i] = tokenFieldCount + i;
}
PermutingTupleReference expectedTuple = new PermutingTupleReference(fieldPermutation);
// Iterate over all tokens. Find the inverted-lists in actual and expected indexes. Compare the inverted lists,
while (tokensIter.hasNext()) {
Comparable token = tokensIter.next();
// Position inverted-list iterator on expected index.
CheckTuple checkLowKey = new CheckTuple(tokenFieldCount, tokenFieldCount);
checkLowKey.appendField(token);
CheckTuple checkHighKey = new CheckTuple(tokenFieldCount, tokenFieldCount);
checkHighKey.appendField(token);
SortedSet<CheckTuple> expectedInvList = OrderedIndexTestUtils.getPrefixExpectedSubset(testCtx.getCheckTuples(), checkLowKey, checkHighKey);
Iterator<CheckTuple> expectedInvListIter = expectedInvList.iterator();
// Position inverted-list cursor in actual index.
OrderedIndexTestUtils.createTupleFromCheckTuple(checkLowKey, searchKeyBuilder, searchKey, fieldSerdes);
invIndexAccessor.openInvertedListCursor(actualInvListCursor, searchKey);
if (actualInvListCursor.size() != expectedInvList.size()) {
fail("Actual and expected inverted lists for token '" + token.toString() + "' have different sizes. Actual size: " + actualInvListCursor.size() + ". Expected size: " + expectedInvList.size() + ".");
}
// Compare inverted-list elements.
int count = 0;
actualInvListCursor.pinPages();
try {
while (actualInvListCursor.hasNext() && expectedInvListIter.hasNext()) {
actualInvListCursor.next();
ITupleReference actual = actualInvListCursor.getTuple();
CheckTuple expected = expectedInvListIter.next();
OrderedIndexTestUtils.createTupleFromCheckTuple(expected, expectedBuilder, completeExpectedTuple, fieldSerdes);
expectedTuple.reset(completeExpectedTuple);
if (invListCmp.compare(actual, expectedTuple) != 0) {
fail("Inverted lists of token '" + token + "' differ at position " + count + ".");
}
count++;
}
} finally {
actualInvListCursor.unpinPages();
}
}
}
Aggregations