use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class BTreeSearchCursorTest method nonUniqueIndexTest.
@Test
public void nonUniqueIndexTest() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
}
IBufferCache bufferCache = harness.getBufferCache();
// declare keys
int keyFieldCount = 2;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
btree.create();
btree.activate();
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
int maxKey = 10;
ArrayList<Integer> keys = new ArrayList<>();
for (int i = 0; i < numKeys; i++) {
int k = rnd.nextInt() % maxKey;
keys.add(k);
}
Collections.sort(keys);
// insert keys into btree
for (int i = 0; i < keys.size(); i++) {
TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
try {
indexAccessor.insert(tuple);
} catch (Exception e) {
e.printStackTrace();
}
}
int minSearchKey = -100;
int maxSearchKey = 100;
// forward searches
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false));
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false));
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
btree.deactivate();
btree.destroy();
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class BTreeSearchCursorTest method uniqueIndexTest.
@Test
public void uniqueIndexTest() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
}
IBufferCache bufferCache = harness.getBufferCache();
// declare keys
int keyFieldCount = 1;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
btree.create();
btree.activate();
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
// generate keys
int numKeys = 50;
int maxKey = 1000;
TreeSet<Integer> uniqueKeys = new TreeSet<>();
ArrayList<Integer> keys = new ArrayList<>();
while (uniqueKeys.size() < numKeys) {
int key = rnd.nextInt() % maxKey;
uniqueKeys.add(key);
}
for (Integer i : uniqueKeys) {
keys.add(i);
}
// insert keys into btree
for (int i = 0; i < keys.size(); i++) {
TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
try {
indexAccessor.insert(tuple);
} catch (Exception e) {
e.printStackTrace();
}
}
int minSearchKey = -100;
int maxSearchKey = 100;
// forward searches
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false));
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false));
Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
btree.deactivate();
btree.destroy();
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class MetadataNode method createExternalFileSearchTuple.
// This method is used to create a search tuple for external data file since the search tuple has an int value
@SuppressWarnings("unchecked")
public ITupleReference createExternalFileSearchTuple(String dataverseName, String datasetName, int fileNumber) throws HyracksDataException {
ISerializerDeserializer<AString> stringSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
ISerializerDeserializer<AInt32> intSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
AMutableString aString = new AMutableString("");
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(3);
//dataverse field
aString.setValue(dataverseName);
stringSerde.serialize(aString, tupleBuilder.getDataOutput());
tupleBuilder.addFieldEndOffset();
//dataset field
aString.setValue(datasetName);
stringSerde.serialize(aString, tupleBuilder.getDataOutput());
tupleBuilder.addFieldEndOffset();
//file number field
intSerde.serialize(new AInt32(fileNumber), tupleBuilder.getDataOutput());
tupleBuilder.addFieldEndOffset();
ArrayTupleReference tuple = new ArrayTupleReference();
tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
return tuple;
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class MetadataNode method createTuple.
// TODO: Can use Hyrack's TupleUtils for this, once we switch to a newer
// Hyracks version.
public static ITupleReference createTuple(String... fields) {
@SuppressWarnings("unchecked") ISerializerDeserializer<AString> stringSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
AMutableString aString = new AMutableString("");
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fields.length);
for (String s : fields) {
aString.setValue(s);
try {
stringSerde.serialize(aString, tupleBuilder.getDataOutput());
} catch (HyracksDataException e) {
// This should never happen
throw new IllegalStateException("Failed to create search tuple!!!! This should never happen", e);
}
tupleBuilder.addFieldEndOffset();
}
ArrayTupleReference tuple = new ArrayTupleReference();
tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
return tuple;
}
use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.
the class AbstractRTreeExamplesTest method deleteExample.
/**
* Deletion Example. Create an RTree index of two dimensions, where they
* keys are of type integer, and the payload is one integer value. Fill
* index with random values using insertions, then delete entries
* one-by-one. Repeat procedure a few times on same RTree.
*/
@Test
public void deleteExample() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Deletion Example");
}
// Declare fields.
int fieldCount = 5;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
typeTraits[0] = IntegerPointable.TYPE_TRAITS;
typeTraits[1] = IntegerPointable.TYPE_TRAITS;
typeTraits[2] = IntegerPointable.TYPE_TRAITS;
typeTraits[3] = IntegerPointable.TYPE_TRAITS;
typeTraits[4] = IntegerPointable.TYPE_TRAITS;
// Declare RTree keys.
int rtreeKeyFieldCount = 4;
IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// Declare BTree keys, this will only be used for LSMRTree
int btreeKeyFieldCount;
IBinaryComparatorFactory[] btreeCmpFactories;
int[] btreeFields = null;
if (rTreeType == RTreeType.LSMRTREE) {
//Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
btreeKeyFieldCount = 1;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeFields = new int[btreeKeyFieldCount];
for (int i = 0; i < btreeKeyFieldCount; i++) {
btreeFields[i] = rtreeKeyFieldCount + i;
}
} else {
btreeKeyFieldCount = 5;
btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
}
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(rtreeCmpFactories.length, IntegerPointable.FACTORY);
ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
treeIndex.create();
treeIndex.activate();
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
int runs = 3;
for (int run = 0; run < runs; run++) {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
LOGGER.info("Inserting into tree...");
}
int numInserts = 10000;
int[] p1xs = new int[numInserts];
int[] p1ys = new int[numInserts];
int[] p2xs = new int[numInserts];
int[] p2ys = new int[numInserts];
int[] pks = new int[numInserts];
int insDone = 0;
int[] insDoneCmp = new int[numInserts];
for (int i = 0; i < numInserts; i++) {
int p1x = rnd.nextInt();
int p1y = rnd.nextInt();
int p2x = rnd.nextInt();
int p2y = rnd.nextInt();
int pk = 5;
p1xs[i] = Math.min(p1x, p2x);
p1ys[i] = Math.min(p1y, p2y);
p2xs[i] = Math.max(p1x, p2x);
p2ys[i] = Math.max(p1y, p2y);
pks[i] = pk;
TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), pk);
try {
indexAccessor.insert(tuple);
} catch (HyracksDataException e) {
if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
throw e;
}
}
insDoneCmp[i] = insDone;
}
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Deleting from tree...");
}
int delDone = 0;
for (int i = 0; i < numInserts; i++) {
TupleUtils.createIntegerTuple(tb, tuple, p1xs[i], p1ys[i], p2xs[i], p2ys[i], pks[i]);
try {
indexAccessor.delete(tuple);
delDone++;
} catch (HyracksDataException e) {
if (e.getErrorCode() != ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY) {
throw e;
}
}
if (insDoneCmp[i] != delDone) {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
}
break;
}
}
if (insDone != delDone) {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
}
break;
}
}
treeIndex.deactivate();
treeIndex.destroy();
}
Aggregations