Examples with TupleFileReader org.bboxdb.tools.TupleFileReader used on opensource projects

Example 1 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class TestFixedGrid method runExperiment.

/**
 * Run this experiment
 * @param cellGrid
 */
protected void runExperiment(final CellGrid cellGrid) {
    final TupleFileReader tupleFile = new TupleFileReader(filename, format);
    final Map<BoundingBox, Integer> bboxes = new HashMap<>();
    final ExecutorService executor = ExecutorUtil.getBoundThreadPoolExecutor(10, 100);
    tupleFile.addTupleListener(t -> {
        executor.submit(() -> {
            final Set<BoundingBox> intersectedBoxes = cellGrid.getAllInersectedBoundingBoxes(t.getBoundingBox());
            synchronized (bboxes) {
                for (final BoundingBox box : intersectedBoxes) {
                    if (bboxes.containsKey(box)) {
                        final int oldValue = bboxes.get(box);
                        bboxes.put(box, oldValue + 1);
                    } else {
                        bboxes.put(box, 1);
                    }
                }
            }
        });
    });
    try {
        System.out.println("# Processing tuples");
        tupleFile.processFile();
        executor.shutdown();
    } catch (IOException e) {
        System.err.println("Got an IOException during experiment: " + e);
        System.exit(-1);
    }
    calculateResult(bboxes);
}

Example 2 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class TestKDTreeSplit method runExperiment.

/**
 * Run the experiment with the given sample size
 * @param sampleSize
 * @throws IOException
 */
protected void runExperiment(final int maxRegionSize) {
    System.out.println("# Simulating with max element size: " + maxRegionSize);
    elements.clear();
    boxDimension.clear();
    for (Entry<String, String> elements : filesAndFormats.entrySet()) {
        final String filename = elements.getKey();
        final String format = elements.getValue();
        System.out.println("Processing file: " + filename);
        final TupleFileReader tupleFile = new TupleFileReader(filename, format);
        tupleFile.addTupleListener(t -> {
            insertNextBoundingBox(t.getBoundingBox(), maxRegionSize);
        });
        try {
            tupleFile.processFile();
        } catch (IOException e) {
            System.err.println("Got an IOException during experiment: " + e);
            System.exit(-1);
        }
    }
    // Print results
    final List<Integer> buckets = elements.values().stream().map(l -> l.size()).collect(Collectors.toList());
    IntStream.range(0, buckets.size()).forEach(i -> System.out.format("%d\t%d\n", i, buckets.get(i)));
}

Example 3 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class CreateInitialPartitioning method run.

@Override
public void run() {
    final TupleFileReader tupleFile = new TupleFileReader(filename, format);
    final List<BoundingBox> samples = new ArrayList<>();
    tupleFile.addTupleListener(t -> {
        final BoundingBox polygonBoundingBox = t.getBoundingBox();
        samples.add(polygonBoundingBox);
    });
    try {
        tupleFile.processFile();
        final SpacePartitioner spacePartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroup);
        final DistributionRegionAdapter adapter = ZookeeperClientFactory.getZookeeperClient().getDistributionRegionAdapter();
        while (getActiveRegions(spacePartitioner).size() < partitions) {
            logger.info("We have now {} of {} active partitons, executing split", getActiveRegions(spacePartitioner).size() < partitions);
            final List<DistributionRegion> activeRegions = getActiveRegions(spacePartitioner);
            final DistributionRegion regionToSplit = ListHelper.getElementRandom(activeRegions);
            logger.info("Splitting region {}", regionToSplit.getRegionId());
            final List<DistributionRegion> destination = spacePartitioner.splitRegion(regionToSplit, samples);
            spacePartitioner.splitComplete(regionToSplit, destination);
        }
        // Prevent merging of nodes
        for (DistributionRegion region : spacePartitioner.getRootNode().getAllChildren()) {
            adapter.setMergingSupported(region, false);
        }
    } catch (Exception e) {
        logger.error("Got an exception", e);
        System.exit(-1);
    }
}

Example 4 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class TestTupleBuilder method testTupleFile3.

/**
 * Test the tuple file builder
 * @throws IOException
 */
@Test
public void testTupleFile3() throws IOException {
    final File tempFile = File.createTempFile("temp", ".txt");
    tempFile.deleteOnExit();
    // The reference tuple
    final TupleBuilder tupleBuilder = TupleBuilderFactory.getBuilderForFormat(TupleBuilderFactory.Name.GEOJSON);
    final Tuple tuple = tupleBuilder.buildTuple("1", GEO_JSON_LINE);
    final BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile));
    writer.write(GEO_JSON_LINE);
    writer.write("\n");
    writer.write(GEO_JSON_LINE);
    writer.write("\n");
    writer.close();
    final TupleFileReader tupleFile = new TupleFileReader(tempFile.getAbsolutePath(), TupleBuilderFactory.Name.GEOJSON);
    final AtomicInteger seenTuples = new AtomicInteger(0);
    tupleFile.addTupleListener(t -> {
        Assert.assertEquals(tuple.getKey(), t.getKey());
        Assert.assertEquals(tuple.getBoundingBox(), t.getBoundingBox());
        Assert.assertArrayEquals(tuple.getDataBytes(), t.getDataBytes());
        seenTuples.incrementAndGet();
    });
    tupleFile.processFile(1);
    Assert.assertEquals(1, seenTuples.get());
    Assert.assertEquals(2, tupleFile.getProcessedLines());
    Assert.assertEquals(GEO_JSON_LINE, tupleFile.getLastReadLine());
}

Example 5 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class TestTupleBuilder method testTupleFile1.

/**
 * Test the tuple file builder - Process non existing file
 * @throws IOException
 */
@Test(expected = IOException.class)
public void testTupleFile1() throws IOException {
    final File tempFile = File.createTempFile("temp", ".txt");
    tempFile.delete();
    Assert.assertFalse(tempFile.exists());
    final TupleFileReader tupleFile = new TupleFileReader(tempFile.getAbsolutePath(), TupleBuilderFactory.Name.GEOJSON);
    tupleFile.processFile();
}