Examples with Tuple org.bboxdb.storage.entity.Tuple used on opensource projects

Example 81 with Tuple

use of org.bboxdb.storage.entity.Tuple in project bboxdb by jnidzwetzki.

the class BenchmarkInsertPerformance method runBenchmark.

@Override
public void runBenchmark() throws InterruptedException, ExecutionException, BBoxDBException {
    // Number of tuples
    final int tuples = 5000000;
    // Remove old data
    final EmptyResultFuture deleteResult = bboxdbClient.deleteDistributionGroup(DISTRIBUTION_GROUP);
    deleteResult.waitForAll();
    // Create a new distribution group
    final DistributionGroupConfiguration config = DistributionGroupConfigurationBuilder.create(3).withReplicationFactor((short) 3).build();
    final EmptyResultFuture createResult = bboxdbClient.createDistributionGroup(DISTRIBUTION_GROUP, config);
    createResult.waitForAll();
    final Random bbBoxRandom = new Random();
    // Insert the tuples
    for (; insertedTuples.get() < tuples; insertedTuples.incrementAndGet()) {
        final double x = Math.abs(bbBoxRandom.nextFloat() % 100000.0 * 1000);
        final double y = Math.abs(bbBoxRandom.nextFloat() % 100000.0 * 1000);
        final double z = Math.abs(bbBoxRandom.nextFloat() % 100000.0 * 1000);
        final BoundingBox boundingBox = new BoundingBox(x, x + 1, y, y + 1, z, z + 1);
        final EmptyResultFuture insertFuture = bboxdbClient.insertTuple(TABLE, new Tuple(Integer.toString(insertedTuples.get()), boundingBox, "abcdef".getBytes()));
        // register pending future
        pendingFutures.put(insertFuture);
    }
}

Example 82 with Tuple

use of org.bboxdb.storage.entity.Tuple in project bboxdb by jnidzwetzki.

the class BenchmarkKeyQueryPerformance method prepare.

@Override
protected void prepare() throws Exception {
    super.prepare();
    // Remove old data
    final EmptyResultFuture deleteResult = bboxdbClient.deleteDistributionGroup(DISTRIBUTION_GROUP);
    deleteResult.waitForAll();
    // Create a new distribution group
    final DistributionGroupConfiguration config = DistributionGroupConfigurationBuilder.create(3).withReplicationFactor((short) 3).build();
    final EmptyResultFuture createResult = bboxdbClient.createDistributionGroup(DISTRIBUTION_GROUP, config);
    createResult.waitForAll();
    logger.info("Inserting {} tuples", tuplesToInsert);
    // Insert the tuples
    for (; insertedTuples.get() < tuplesToInsert; insertedTuples.incrementAndGet()) {
        bboxdbClient.insertTuple(TABLE, new Tuple(Integer.toString(insertedTuples.get()), BoundingBox.FULL_SPACE, "abcdef".getBytes()));
    }
    // Wait for requests to settle
    logger.info("Wait for insert requests to settle");
    while (bboxdbClient.getInFlightCalls() != 0) {
        logger.info("{} tuples are pending", bboxdbClient.getInFlightCalls());
        Thread.sleep(1000);
    }
    logger.info("All insert requests are settled");
}

Example 83 with Tuple

use of org.bboxdb.storage.entity.Tuple in project bboxdb by jnidzwetzki.

the class DataRedistributionLoader method loadFile.

/**
 * Load the given file
 * @param id
 * @return
 * @throws InterruptedException
 */
private boolean loadFile(final int fileid) throws InterruptedException {
    final String filename = files[fileid];
    if (loadedFiles.contains(filename)) {
        System.err.println("File " + filename + " is already loaded");
        return false;
    }
    System.out.println("Loading content from: " + filename);
    final AtomicInteger lineNumber = new AtomicInteger(0);
    final String prefix = Integer.toString(fileid) + "_";
    try (final Stream<String> lines = Files.lines(Paths.get(filename))) {
        lines.forEach(l -> {
            final String key = prefix + lineNumber.getAndIncrement();
            final Tuple tuple = tupleBuilder.buildTuple(key, l);
            try {
                if (tuple != null) {
                    final EmptyResultFuture insertFuture = bboxDBCluster.insertTuple(TABLE, tuple);
                    pendingFutures.put(insertFuture);
                }
            } catch (BBoxDBException e) {
                logger.error("Got error while inserting tuple", e);
            }
            if (lineNumber.get() % 1000 == 0) {
                System.out.format("Loaded %d elements\n", lineNumber.get());
            }
        });
    } catch (IOException e) {
        System.err.println("Got an exeption while reading file: " + e);
        System.exit(-1);
    }
    pendingFutures.waitForCompletion();
    loadedFiles.add(filename);
    System.out.println("Loaded content from: " + filename);
    return true;
}

Example 84 with Tuple

use of org.bboxdb.storage.entity.Tuple in project bboxdb by jnidzwetzki.

the class SamplingHelper method processTupleStores.

/**
 * Process the facades for the table and create samples
 * @param storages
 * @param splitDimension
 * @param boundingBox
 * @param floatIntervals
 * @return
 * @throws StorageManagerException
 */
private static List<BoundingBox> processTupleStores(final List<ReadOnlyTupleStore> storages) throws StorageManagerException {
    final int samplesPerStorage = 100;
    final List<BoundingBox> samples = new ArrayList<>();
    logger.debug("Fetching {} samples per storage", samplesPerStorage);
    for (final ReadOnlyTupleStore storage : storages) {
        if (!storage.acquire()) {
            continue;
        }
        final long numberOfTuples = storage.getNumberOfTuples();
        final int sampleOffset = Math.max(10, (int) (numberOfTuples / samplesPerStorage));
        for (long position = 0; position < numberOfTuples; position = position + sampleOffset) {
            final Tuple tuple = storage.getTupleAtPosition(position);
            final BoundingBox tupleBoundingBox = tuple.getBoundingBox();
            // Ignore tuples with an empty box (e.g. deleted tuples)
            if (tupleBoundingBox == null || tupleBoundingBox.equals(BoundingBox.FULL_SPACE)) {
                continue;
            }
            samples.add(tupleBoundingBox);
        }
        storage.release();
    }
    return samples;
}

Example 85 with Tuple

use of org.bboxdb.storage.entity.Tuple in project bboxdb by jnidzwetzki.

the class TestSSTableBloomFilter method writeTuples.

/**
 * Write the tuples
 * @param data
 * @param tupleStore
 * @return
 * @throws IOException
 */
protected long writeTuples(final String data, SSTableTupleStore tupleStore) throws Exception {
    System.out.println("# Writing Tuples");
    final Stopwatch stopwatch = Stopwatch.createStarted();
    for (int i = 0; i < TUPLES; i++) {
        final Tuple tuple = new Tuple(Integer.toString(i), BoundingBox.FULL_SPACE, data.getBytes());
        tupleStore.writeTuple(tuple);
    }
    return stopwatch.elapsed(TimeUnit.MILLISECONDS);
}