use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class AbstractRTreeExamplesTest method rStarTreePageSplitTestExample.
/**
* This test the r*tree page split. Originally this test didn't pass since
* the r*tree assumes always that there will be enough space for the new
* tuple after split. Now it passes since if there is not space in the
* designated page, then we will just insert it in the other split page.
*/
@Test
public void rStarTreePageSplitTestExample() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("R*Tree page split test.");
}
// Declare fields.
int fieldCount = 5;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
typeTraits[0] = IntegerPointable.TYPE_TRAITS;
typeTraits[1] = IntegerPointable.TYPE_TRAITS;
typeTraits[2] = IntegerPointable.TYPE_TRAITS;
typeTraits[3] = IntegerPointable.TYPE_TRAITS;
typeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
// Declare field serdes.
ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
// Declare RTree keys.
int rtreeKeyFieldCount = 4;
IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// Declare BTree keys, this will only be used for LSMRTree
int btreeKeyFieldCount;
IBinaryComparatorFactory[] btreeCmpFactories;
int[] btreeFields = null;
if (rTreeType == RTreeType.LSMRTREE) {
//Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
btreeKeyFieldCount = 1;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
btreeFields = new int[btreeKeyFieldCount];
for (int i = 0; i < btreeKeyFieldCount; i++) {
btreeFields[i] = rtreeKeyFieldCount + i;
}
} else {
btreeKeyFieldCount = 5;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
}
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, RTreePolicyType.RSTARTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
int p1x = rnd.nextInt();
int p1y = rnd.nextInt();
int p2x = rnd.nextInt();
int p2y = rnd.nextInt();
String data = "";
for (int i = 0; i < 210; i++) {
data += "X";
}
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "";
for (int i = 0; i < 210; i++) {
data += "X";
}
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "XXX";
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "XXX";
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "";
for (int i = 0; i < 210; i++) {
data += "X";
}
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "";
for (int i = 0; i < 210; i++) {
data += "X";
}
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
treeIndex.deactivate();
treeIndex.destroy();
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class AbstractRTreeExamplesTest method threeDimensionsExample.
/**
* Two Dimensions Example. Create an RTree index of three dimensions, where
* they keys are of type double, and the payload is one double value. Fill
* index with random values using insertions (not bulk load). Perform scans
* and range search.
*/
@Test
public void threeDimensionsExample() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Fixed-Length Key,Value Example.");
}
// Declare fields.
int fieldCount = 7;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
typeTraits[0] = DoublePointable.TYPE_TRAITS;
typeTraits[1] = DoublePointable.TYPE_TRAITS;
typeTraits[2] = DoublePointable.TYPE_TRAITS;
typeTraits[3] = DoublePointable.TYPE_TRAITS;
typeTraits[4] = DoublePointable.TYPE_TRAITS;
typeTraits[5] = DoublePointable.TYPE_TRAITS;
typeTraits[6] = DoublePointable.TYPE_TRAITS;
// Declare field serdes.
ISerializerDeserializer[] fieldSerdes = { DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
// Declare RTree keys.
int rtreeKeyFieldCount = 6;
IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
rtreeCmpFactories[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
rtreeCmpFactories[5] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
// Declare BTree keys, this will only be used for LSMRTree
int btreeKeyFieldCount;
IBinaryComparatorFactory[] btreeCmpFactories;
int[] btreeFields = null;
if (rTreeType == RTreeType.LSMRTREE) {
//Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
btreeKeyFieldCount = 1;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
btreeFields = new int[btreeKeyFieldCount];
for (int i = 0; i < btreeKeyFieldCount; i++) {
btreeFields[i] = rtreeKeyFieldCount + i;
}
} else {
btreeKeyFieldCount = 7;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
btreeCmpFactories[6] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
}
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(rtreeCmpFactories.length, DoublePointable.FACTORY);
//4
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
long start = System.currentTimeMillis();
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Inserting into tree...");
}
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
int numInserts = 10000;
for (int i = 0; i < numInserts; i++) {
double p1x = rnd.nextDouble();
double p1y = rnd.nextDouble();
double p1z = rnd.nextDouble();
double p2x = rnd.nextDouble();
double p2y = rnd.nextDouble();
double p2z = rnd.nextDouble();
double pk = 5.0;
TupleUtils.createDoubleTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.min(p1z, p2z), Math.max(p1x, p2x), Math.max(p1y, p2y), Math.max(p1z, p2z), pk);
try {
indexAccessor.insert(tuple);
} catch (HyracksDataException e) {
if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
throw e;
}
}
}
long end = System.currentTimeMillis();
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
}
scan(indexAccessor, fieldSerdes);
diskOrderScan(indexAccessor, fieldSerdes);
// Build key.
ArrayTupleBuilder keyTb = new ArrayTupleBuilder(rtreeKeyFieldCount);
ArrayTupleReference key = new ArrayTupleReference();
TupleUtils.createDoubleTuple(keyTb, key, -1000.0, -1000.0, -1000.0, 1000.0, 1000.0, 1000.0);
rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key, null, null);
treeIndex.deactivate();
treeIndex.destroy();
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class AbstractRTreeExamplesTest method rTreePageSplitTestExample.
/**
* This test the rtree page split. Originally this test didn't pass since
* the rtree assumes always that there will be enough space for the new
* tuple after split. Now it passes since if there is not space in the
* designated page, then we will just insert it in the other split page.
*/
@Test
public void rTreePageSplitTestExample() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("RTree page split test.");
}
// Declare fields.
int fieldCount = 5;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
typeTraits[0] = IntegerPointable.TYPE_TRAITS;
typeTraits[1] = IntegerPointable.TYPE_TRAITS;
typeTraits[2] = IntegerPointable.TYPE_TRAITS;
typeTraits[3] = IntegerPointable.TYPE_TRAITS;
typeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
// Declare field serdes.
ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
// Declare RTree keys.
int rtreeKeyFieldCount = 4;
IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// Declare BTree keys, this will only be used for LSMRTree
int btreeKeyFieldCount;
IBinaryComparatorFactory[] btreeCmpFactories;
int[] btreeFields = null;
if (rTreeType == RTreeType.LSMRTREE) {
//Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
btreeKeyFieldCount = 1;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
btreeFields = new int[btreeKeyFieldCount];
for (int i = 0; i < btreeKeyFieldCount; i++) {
btreeFields[i] = rtreeKeyFieldCount + i;
}
} else {
btreeKeyFieldCount = 5;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
}
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(rtreeCmpFactories.length, IntegerPointable.FACTORY);
//2
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
int p1x = rnd.nextInt();
int p1y = rnd.nextInt();
int p2x = rnd.nextInt();
int p2y = rnd.nextInt();
String data = "";
for (int i = 0; i < 210; i++) {
data += "X";
}
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "XXX";
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "XXX";
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "XXX";
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "";
for (int i = 0; i < 210; i++) {
data += "X";
}
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
p1x = rnd.nextInt();
p1y = rnd.nextInt();
p2x = rnd.nextInt();
p2y = rnd.nextInt();
data = "";
for (int i = 0; i < 210; i++) {
data += "X";
}
TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
indexAccessor.insert(tuple);
treeIndex.deactivate();
treeIndex.destroy();
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class OrderedIndexTestUtils method checkPointSearches.
public void checkPointSearches(IIndexTestContext ictx) throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Testing Point Searches On All Expected Keys.");
}
OrderedIndexTestContext ctx = (OrderedIndexTestContext) ictx;
IIndexCursor searchCursor = ctx.getIndexAccessor().createSearchCursor(false);
ArrayTupleBuilder lowKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
ArrayTupleReference lowKey = new ArrayTupleReference();
ArrayTupleBuilder highKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
ArrayTupleReference highKey = new ArrayTupleReference();
RangePredicate rangePred = new RangePredicate(lowKey, highKey, true, true, null, null);
// BTree to verify the tuple can be reached.
for (CheckTuple checkTuple : ctx.getCheckTuples()) {
createTupleFromCheckTuple(checkTuple, lowKeyBuilder, lowKey, ctx.getFieldSerdes());
createTupleFromCheckTuple(checkTuple, highKeyBuilder, highKey, ctx.getFieldSerdes());
MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), lowKey);
MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), highKey);
rangePred.setLowKey(lowKey, true);
rangePred.setHighKey(highKey, true);
rangePred.setLowKeyComparator(lowKeyCmp);
rangePred.setHighKeyComparator(highKeyCmp);
ctx.getIndexAccessor().search(searchCursor, rangePred);
try {
// We expect exactly one answer.
if (searchCursor.hasNext()) {
searchCursor.next();
ITupleReference tuple = searchCursor.getTuple();
compareActualAndExpected(tuple, checkTuple, ctx.getFieldSerdes());
}
if (searchCursor.hasNext()) {
fail("Point search returned more than one answer.");
}
} finally {
searchCursor.close();
}
}
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class TreeIndexTestUtils method deleteTuples.
@SuppressWarnings("unchecked")
public void deleteTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
ArrayTupleBuilder deleteTupleBuilder = createDeleteTupleBuilder(ctx);
ArrayTupleReference deleteTuple = new ArrayTupleReference();
int numCheckTuples = ctx.getCheckTuples().size();
// Copy CheckTuple references into array, so we can randomly pick from
// there.
CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
int idx = 0;
Iterator<CheckTuple> iter = ctx.getCheckTuples().iterator();
while (iter.hasNext()) {
CheckTuple checkTuple = iter.next();
checkTuples[idx++] = checkTuple;
}
for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
if (LOGGER.isLoggable(Level.INFO)) {
if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
LOGGER.info("Deleting Tuple " + (i + 1) + "/" + numTuples);
}
}
int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
CheckTuple checkTuple = checkTuples[checkTupleIdx];
createTupleFromCheckTuple(checkTuple, deleteTupleBuilder, deleteTuple, ctx.getFieldSerdes());
ctx.getIndexAccessor().delete(deleteTuple);
// Remove check tuple from expected results.
ctx.deleteCheckTuple(checkTuple, ctx.getCheckTuples());
// Swap with last "valid" CheckTuple.
CheckTuple tmp = checkTuples[numCheckTuples - 1];
checkTuples[numCheckTuples - 1] = checkTuple;
checkTuples[checkTupleIdx] = tmp;
numCheckTuples--;
}
}
Aggregations