Examples with JsonIterator org.apache.druid.data.input.impl.prefetch.JsonIterator used on opensource projects

Example 1 with JsonIterator

use of org.apache.druid.data.input.impl.prefetch.JsonIterator in project druid by druid-io.

the class PrefetchSqlFirehoseFactory method connect.

@Override
public Firehose connect(InputRowParser<Map<String, Object>> firehoseParser, @Nullable File temporaryDirectory) {
    if (objects == null) {
        objects = ImmutableList.copyOf(Preconditions.checkNotNull(initObjects(), "objects"));
    }
    if (cacheManager.isEnabled() || fetchConfig.getMaxFetchCapacityBytes() > 0) {
        Preconditions.checkNotNull(temporaryDirectory, "temporaryDirectory");
        Preconditions.checkArgument(temporaryDirectory.exists(), "temporaryDirectory[%s] does not exist", temporaryDirectory);
        Preconditions.checkArgument(temporaryDirectory.isDirectory(), "temporaryDirectory[%s] is not a directory", temporaryDirectory);
    }
    LOG.info("Create a new firehose for [%d] queries", objects.size());
    // fetchExecutor is responsible for background data fetching
    final ExecutorService fetchExecutor = Execs.singleThreaded("firehose_fetch_%d");
    final Fetcher<T> fetcher = new SqlFetcher<>(cacheManager, objects, fetchExecutor, temporaryDirectory, fetchConfig, new ObjectOpenFunction<T>() {

        @Override
        public InputStream open(T object, File outFile) throws IOException {
            return openObjectStream(object, outFile);
        }

        @Override
        public InputStream open(T object) throws IOException {
            final File outFile = File.createTempFile("sqlresults_", null, temporaryDirectory);
            return openObjectStream(object, outFile);
        }
    });
    return new SqlFirehose(new Iterator<JsonIterator<Map<String, Object>>>() {

        @Override
        public boolean hasNext() {
            return fetcher.hasNext();
        }

        @Override
        public JsonIterator<Map<String, Object>> next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }
            try {
                TypeReference<Map<String, Object>> type = new TypeReference<Map<String, Object>>() {
                };
                final OpenObject<T> openObject = fetcher.next();
                final InputStream stream = openObject.getObjectStream();
                return new JsonIterator<>(type, stream, openObject.getResourceCloser(), objectMapper);
            } catch (Exception ioe) {
                throw new RuntimeException(ioe);
            }
        }
    }, firehoseParser, () -> {
        fetchExecutor.shutdownNow();
        try {
            Preconditions.checkState(fetchExecutor.awaitTermination(fetchConfig.getFetchTimeout(), TimeUnit.MILLISECONDS));
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ISE("Failed to shutdown fetch executor during close");
        }
    });
}

Example 2 with JsonIterator

use of org.apache.druid.data.input.impl.prefetch.JsonIterator in project druid by druid-io.

the class SqlFirehoseTest method testFirehoseStringParser.

@Test
public void testFirehoseStringParser() throws Exception {
    final TestCloseable closeable = new TestCloseable();
    List<Object> expectedResults = new ArrayList<>();
    for (Map<String, Object> map : inputs) {
        expectedResults.add(map.get("x"));
    }
    final List<JsonIterator<Map<String, Object>>> lineIterators = fileList.stream().map(s -> new JsonIterator<Map<String, Object>>(TYPE_REF, s, closeable, objectMapper)).collect(Collectors.toList());
    final InputRowParser stringParser = TransformSpec.NONE.decorate(new StringInputRowParser(new TimeAndDimsParseSpec(new TimestampSpec("timestamp", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("x")))), Charset.defaultCharset().name()));
    try (final SqlFirehose firehose = new SqlFirehose(lineIterators.iterator(), stringParser, closeable)) {
        final List<Object> results = new ArrayList<>();
        while (firehose.hasMore()) {
            final InputRow inputRow = firehose.nextRow();
            if (inputRow == null) {
                results.add(null);
            } else {
                results.add(inputRow.getDimension("x").get(0));
            }
        }
        Assert.assertEquals(expectedResults, results);
    }
}

Example 3 with JsonIterator

use of org.apache.druid.data.input.impl.prefetch.JsonIterator in project druid by druid-io.

the class SqlFirehoseTest method testFirehose.

@Test
public void testFirehose() throws Exception {
    final TestCloseable closeable = new TestCloseable();
    List<Object> expectedResults = new ArrayList<>();
    for (Map<String, Object> map : inputs) {
        expectedResults.add(map.get("x"));
    }
    final List<JsonIterator<Map<String, Object>>> lineIterators = fileList.stream().map(s -> new JsonIterator<Map<String, Object>>(TYPE_REF, s, closeable, objectMapper)).collect(Collectors.toList());
    try (final SqlFirehose firehose = new SqlFirehose(lineIterators.iterator(), parser, closeable)) {
        final List<Object> results = new ArrayList<>();
        while (firehose.hasMore()) {
            final InputRow inputRow = firehose.nextRow();
            if (inputRow == null) {
                results.add(null);
            } else {
                results.add(inputRow.getDimension("x").get(0));
            }
        }
        Assert.assertEquals(expectedResults, results);
    }
}

Example 4 with JsonIterator

use of org.apache.druid.data.input.impl.prefetch.JsonIterator in project druid by druid-io.

the class SqlFirehoseTest method testClose.

@Test
public void testClose() throws IOException {
    File file = File.createTempFile("test", "", TEST_DIR);
    final TestCloseable closeable = new TestCloseable();
    try (FileOutputStream fos = new FileOutputStream(file)) {
        final JsonGenerator jg = objectMapper.getFactory().createGenerator(fos);
        jg.writeStartArray();
        jg.writeEndArray();
        jg.close();
    }
    final JsonIterator<Map<String, Object>> jsonIterator = new JsonIterator<>(TYPE_REF, new FileInputStream(file), closeable, objectMapper);
    final SqlFirehose firehose = new SqlFirehose(ImmutableList.of(jsonIterator).iterator(), parser, closeable);
    // initialize lineIterator
    firehose.hasMore();
    firehose.close();
    Assert.assertTrue(closeable.closed);
}

Example 5 with JsonIterator

use of org.apache.druid.data.input.impl.prefetch.JsonIterator in project druid by druid-io.

the class SqlReader method intermediateRowIterator.

@Override
protected CloseableIterator<Map<String, Object>> intermediateRowIterator() throws IOException {
    final Closer closer = Closer.create();
    // The results are fetched into local storage as this avoids having to keep a persistent database connection for a long time
    final InputEntity.CleanableFile resultFile = closer.register(source.fetch(temporaryDirectory, null));
    FileInputStream inputStream = new FileInputStream(resultFile.file());
    JsonIterator<Map<String, Object>> jsonIterator = new JsonIterator<>(new TypeReference<Map<String, Object>>() {
    }, inputStream, closer, objectMapper);
    return jsonIterator;
}