Examples with TupleFileReader org.bboxdb.tools.TupleFileReader used on opensource projects

Example 6 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class CLI method actionImportData.

/**
 * Import data
 * @param line
 */
protected void actionImportData(final CommandLine line) {
    final List<String> requiredArgs = Arrays.asList(CLIParameter.FILE, CLIParameter.FORMAT, CLIParameter.TABLE);
    checkRequiredArgs(requiredArgs);
    final String filename = line.getOptionValue(CLIParameter.FILE);
    final String format = line.getOptionValue(CLIParameter.FORMAT);
    final String table = line.getOptionValue(CLIParameter.TABLE);
    System.out.println("Importing file: " + filename);
    final TupleFileReader tupleFile = new TupleFileReader(filename, format);
    tupleFile.addTupleListener(t -> {
        if (t == null) {
            logger.error("Unable to parse line: " + tupleFile.getLastReadLine());
            return;
        }
        if (tupleFile.getProcessedLines() % 1000 == 0) {
            System.out.format("Read %d lines%n", tupleFile.getProcessedLines());
        }
        try {
            final EmptyResultFuture result = bboxDbConnection.insertTuple(table, t);
            pendingFutures.put(result);
        } catch (BBoxDBException e) {
            logger.error("Got exception while inserting tuple", e);
        }
    });
    try {
        tupleFile.processFile();
        pendingFutures.waitForCompletion();
        System.out.format("Successfully imported %d lines%n", tupleFile.getProcessedLines());
    } catch (IOException e) {
        logger.error("Got IO Exception while reading data", e);
        System.exit(-1);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        return;
    }
}

Example 7 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class ExperimentStatistics method runExperimentForPos.

/**
 * Run the experiment for the given position
 * @param splitPos
 * @return
 * @throws IOException
 * @throws ClassNotFoundException
 */
protected ExperimentStatistics runExperimentForPos(final double splitPos) throws ClassNotFoundException, IOException {
    final ExperimentStatistics statistics = new ExperimentStatistics();
    final BoundingBox fullBox = BoundingBox.createFullCoveringDimensionBoundingBox(tupleDimension);
    final BoundingBox leftBox = fullBox.splitAndGetLeft(splitPos, 0, true);
    final BoundingBox rightBox = fullBox.splitAndGetRight(splitPos, 0, false);
    final TupleFileReader tupleFile = new TupleFileReader(filename, format);
    tupleFile.addTupleListener(t -> {
        final BoundingBox polygonBoundingBox = t.getBoundingBox();
        boolean tupleDistributed = false;
        if (polygonBoundingBox.overlaps(leftBox)) {
            statistics.increaseLeft();
            tupleDistributed = true;
        }
        if (polygonBoundingBox.overlaps(rightBox)) {
            statistics.increaseRight();
            tupleDistributed = true;
        }
        statistics.increaseTotal();
        if (!tupleDistributed) {
            System.err.println("Unable to distribute: ");
            System.err.println("Left box: " + leftBox);
            System.err.println("Right box: " + rightBox);
            System.err.println("Tuple box: " + polygonBoundingBox);
        }
    });
    try {
        tupleFile.processFile(MAX_ELEMENTS);
    } catch (IOException e) {
        logger.error("Got an IO-Exception while reading file", e);
        System.exit(-1);
    }
    return statistics;
}

Example 8 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class ExperimentHelper method determineBoundingBox.

/**
 * Determine global bounding box
 * @param sampleSize
 * @return
 * @throws IOException
 */
public static BoundingBox determineBoundingBox(final String filename, final String format) {
    System.out.println("# Determining the bounding box");
    final TupleFileReader tupleFile = new TupleFileReader(filename, format);
    final List<BoundingBox> bboxes = new ArrayList<>();
    tupleFile.addTupleListener(t -> {
        bboxes.add(t.getBoundingBox());
    });
    try {
        tupleFile.processFile();
    } catch (IOException e) {
        System.err.println("Got an IOException during experiment: " + e);
        System.exit(-1);
    }
    return BoundingBox.getCoveringBox(bboxes);
}

Example 9 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class TestCompressionRatio method runExperiment.

/**
 * Run the experiment with the given batch size
 * @param batchSize
 * @return
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws PackageEncodeException
 */
protected long runExperiment(final Integer batchSize) throws ClassNotFoundException, IOException, PackageEncodeException {
    final TupleStoreName tableName = new TupleStoreName("2_group1_table1");
    final List<Long> experimentSize = new ArrayList<>();
    final List<Tuple> buffer = new ArrayList<>();
    final TupleFileReader tupleFile = new TupleFileReader(filename, format);
    tupleFile.addTupleListener(t -> {
        if (batchSize == 0) {
            final long size = handleUncompressedData(tableName, t);
            experimentSize.add(size);
        } else if (buffer.size() == batchSize) {
            final long size = handleCompressedData(tableName, buffer);
            experimentSize.add(size);
        } else {
            buffer.add(t);
        }
    });
    try {
        tupleFile.processFile();
    } catch (IOException e) {
        logger.error("Got an IO-Exception while reading file", e);
        System.exit(-1);
    }
    return experimentSize.stream().mapToLong(i -> i).sum();
}

Example 10 with TupleFileReader

use of org.bboxdb.tools.TupleFileReader in project bboxdb by jnidzwetzki.

the class TestTupleBuilder method testTupleFile2.

/**
 * Test the tuple file builder
 * @throws IOException
 */
@Test
public void testTupleFile2() throws IOException {
    final File tempFile = File.createTempFile("temp", ".txt");
    tempFile.deleteOnExit();
    // The reference tuple
    final TupleBuilder tupleBuilder = TupleBuilderFactory.getBuilderForFormat(TupleBuilderFactory.Name.GEOJSON);
    final Tuple tuple = tupleBuilder.buildTuple("1", GEO_JSON_LINE);
    final BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile));
    writer.write(GEO_JSON_LINE);
    writer.write("\n");
    writer.close();
    final TupleFileReader tupleFile = new TupleFileReader(tempFile.getAbsolutePath(), TupleBuilderFactory.Name.GEOJSON);
    final AtomicInteger seenTuples = new AtomicInteger(0);
    tupleFile.addTupleListener(t -> {
        Assert.assertEquals(tuple.getKey(), t.getKey());
        Assert.assertEquals(tuple.getBoundingBox(), t.getBoundingBox());
        Assert.assertArrayEquals(tuple.getDataBytes(), t.getDataBytes());
        seenTuples.incrementAndGet();
    });
    tupleFile.processFile();
    Assert.assertEquals(1, seenTuples.get());
}