Search in sources :

Example 1 with ORDERED

use of java.util.Spliterator.ORDERED in project hazelcast by hazelcast.

the class SqlCompactTest method randomEntryFrom.

@SuppressWarnings({ "OptionalGetWithoutIsPresent", "unchecked", "rawtypes" })
private static Entry<Data, Data> randomEntryFrom(String mapName) {
    NodeEngine engine = getNodeEngine(instance());
    MapService service = engine.getService(MapService.SERVICE_NAME);
    MapServiceContext context = service.getMapServiceContext();
    return Arrays.stream(context.getPartitionContainers()).map(partitionContainer -> partitionContainer.getExistingRecordStore(mapName)).filter(Objects::nonNull).flatMap(store -> {
        Iterator<Entry<Data, Record>> iterator = store.iterator();
        return stream(spliteratorUnknownSize(iterator, ORDERED), false);
    }).map(entry -> entry(entry.getKey(), (Data) entry.getValue().getValue())).findFirst().get();
}
Also used : NodeEngine(com.hazelcast.spi.impl.NodeEngine) Util.getNodeEngine(com.hazelcast.jet.impl.util.Util.getNodeEngine) Spliterators.spliteratorUnknownSize(java.util.Spliterators.spliteratorUnknownSize) Arrays(java.util.Arrays) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) ORDERED(java.util.Spliterator.ORDERED) Collections.singletonList(java.util.Collections.singletonList) BigDecimal(java.math.BigDecimal) Arrays.asList(java.util.Arrays.asList) LocalTime(java.time.LocalTime) CompactSerializer(com.hazelcast.nio.serialization.compact.CompactSerializer) OPTION_KEY_CLASS(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_CLASS) OPTION_KEY_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_FORMAT) SqlRow(com.hazelcast.sql.SqlRow) HazelcastSqlException(com.hazelcast.sql.HazelcastSqlException) GenericRecord(com.hazelcast.nio.serialization.GenericRecord) Objects(java.util.Objects) OPTION_VALUE_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_VALUE_FORMAT) Util(com.hazelcast.jet.impl.util.Util) OPTION_VALUE_COMPACT_TYPE_NAME(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_VALUE_COMPACT_TYPE_NAME) OffsetDateTime(java.time.OffsetDateTime) StreamSupport.stream(java.util.stream.StreamSupport.stream) TestAllTypesSqlConnector(com.hazelcast.jet.sql.impl.connector.test.TestAllTypesSqlConnector) Assert.assertFalse(org.junit.Assert.assertFalse) LocalDate(java.time.LocalDate) Entry(java.util.Map.Entry) UTC(java.time.ZoneOffset.UTC) BeforeClass(org.junit.BeforeClass) LocalDateTime(java.time.LocalDateTime) CompactWriter(com.hazelcast.nio.serialization.compact.CompactWriter) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) InternalSerializationService(com.hazelcast.internal.serialization.InternalSerializationService) Util.entry(com.hazelcast.jet.Util.entry) COMPACT_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.COMPACT_FORMAT) ClientConfig(com.hazelcast.client.config.ClientConfig) Nonnull(javax.annotation.Nonnull) InternalGenericRecord(com.hazelcast.internal.serialization.impl.InternalGenericRecord) Config(com.hazelcast.config.Config) SqlService(com.hazelcast.sql.SqlService) CompactReader(com.hazelcast.nio.serialization.compact.CompactReader) NodeEngine(com.hazelcast.spi.impl.NodeEngine) Iterator(java.util.Iterator) Record(com.hazelcast.map.impl.record.Record) Data(com.hazelcast.internal.serialization.Data) SqlTestSupport(com.hazelcast.jet.sql.SqlTestSupport) Test(org.junit.Test) IOException(java.io.IOException) MapService(com.hazelcast.map.impl.MapService) MapServiceContext(com.hazelcast.map.impl.MapServiceContext) OPTION_KEY_COMPACT_TYPE_NAME(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_COMPACT_TYPE_NAME) Util.getNodeEngine(com.hazelcast.jet.impl.util.Util.getNodeEngine) JAVA_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.JAVA_FORMAT) CompactSerializationConfig(com.hazelcast.config.CompactSerializationConfig) GenericRecordBuilder(com.hazelcast.nio.serialization.GenericRecordBuilder) Assert.assertEquals(org.junit.Assert.assertEquals) IMap(com.hazelcast.map.IMap) Objects(java.util.Objects) Iterator(java.util.Iterator) Data(com.hazelcast.internal.serialization.Data) GenericRecord(com.hazelcast.nio.serialization.GenericRecord) InternalGenericRecord(com.hazelcast.internal.serialization.impl.InternalGenericRecord) Record(com.hazelcast.map.impl.record.Record) MapService(com.hazelcast.map.impl.MapService) MapServiceContext(com.hazelcast.map.impl.MapServiceContext)

Example 2 with ORDERED

use of java.util.Spliterator.ORDERED in project hazelcast by hazelcast.

the class SqlPortableTest method randomEntryFrom.

@SuppressWarnings({ "OptionalGetWithoutIsPresent", "unchecked", "rawtypes" })
private static Entry<Data, Data> randomEntryFrom(String mapName) {
    NodeEngine engine = getNodeEngine(instance());
    MapService service = engine.getService(MapService.SERVICE_NAME);
    MapServiceContext context = service.getMapServiceContext();
    return Arrays.stream(context.getPartitionContainers()).map(partitionContainer -> partitionContainer.getExistingRecordStore(mapName)).filter(Objects::nonNull).flatMap(store -> {
        Iterator<Entry<Data, Record>> iterator = store.iterator();
        return stream(spliteratorUnknownSize(iterator, ORDERED), false);
    }).map(entry -> entry(entry.getKey(), (Data) entry.getValue().getValue())).findFirst().get();
}
Also used : NodeEngine(com.hazelcast.spi.impl.NodeEngine) Util.getNodeEngine(com.hazelcast.jet.impl.util.Util.getNodeEngine) Spliterators.spliteratorUnknownSize(java.util.Spliterators.spliteratorUnknownSize) Arrays(java.util.Arrays) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) PortableGenericRecordBuilder(com.hazelcast.internal.serialization.impl.portable.PortableGenericRecordBuilder) ORDERED(java.util.Spliterator.ORDERED) ClassDefinition(com.hazelcast.nio.serialization.ClassDefinition) Collections.singletonList(java.util.Collections.singletonList) BigDecimal(java.math.BigDecimal) ClassDefinitionBuilder(com.hazelcast.nio.serialization.ClassDefinitionBuilder) Arrays.asList(java.util.Arrays.asList) LocalTime(java.time.LocalTime) OPTION_KEY_CLASS(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_CLASS) OPTION_KEY_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_FORMAT) SqlRow(com.hazelcast.sql.SqlRow) PORTABLE_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.PORTABLE_FORMAT) HazelcastSqlException(com.hazelcast.sql.HazelcastSqlException) OPTION_KEY_CLASS_ID(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_CLASS_ID) GenericRecord(com.hazelcast.nio.serialization.GenericRecord) Collections.emptyList(java.util.Collections.emptyList) Objects(java.util.Objects) OPTION_VALUE_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_VALUE_FORMAT) Util(com.hazelcast.jet.impl.util.Util) OffsetDateTime(java.time.OffsetDateTime) StreamSupport.stream(java.util.stream.StreamSupport.stream) TestAllTypesSqlConnector(com.hazelcast.jet.sql.impl.connector.test.TestAllTypesSqlConnector) Assert.assertFalse(org.junit.Assert.assertFalse) LocalDate(java.time.LocalDate) Entry(java.util.Map.Entry) UTC(java.time.ZoneOffset.UTC) OPTION_KEY_FACTORY_ID(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_FACTORY_ID) OPTION_VALUE_FACTORY_ID(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_VALUE_FACTORY_ID) BeforeClass(org.junit.BeforeClass) LocalDateTime(java.time.LocalDateTime) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) InternalSerializationService(com.hazelcast.internal.serialization.InternalSerializationService) Util.entry(com.hazelcast.jet.Util.entry) InternalGenericRecord(com.hazelcast.internal.serialization.impl.InternalGenericRecord) SqlService(com.hazelcast.sql.SqlService) NodeEngine(com.hazelcast.spi.impl.NodeEngine) Iterator(java.util.Iterator) OPTION_KEY_CLASS_VERSION(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_CLASS_VERSION) Record(com.hazelcast.map.impl.record.Record) Data(com.hazelcast.internal.serialization.Data) SqlTestSupport(com.hazelcast.jet.sql.SqlTestSupport) Test(org.junit.Test) IOException(java.io.IOException) MapService(com.hazelcast.map.impl.MapService) MapServiceContext(com.hazelcast.map.impl.MapServiceContext) OPTION_VALUE_CLASS_VERSION(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_VALUE_CLASS_VERSION) Util.getNodeEngine(com.hazelcast.jet.impl.util.Util.getNodeEngine) OPTION_VALUE_CLASS_ID(com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_VALUE_CLASS_ID) JAVA_FORMAT(com.hazelcast.jet.sql.impl.connector.SqlConnector.JAVA_FORMAT) Assert.assertEquals(org.junit.Assert.assertEquals) Objects(java.util.Objects) Iterator(java.util.Iterator) Data(com.hazelcast.internal.serialization.Data) GenericRecord(com.hazelcast.nio.serialization.GenericRecord) InternalGenericRecord(com.hazelcast.internal.serialization.impl.InternalGenericRecord) Record(com.hazelcast.map.impl.record.Record) MapService(com.hazelcast.map.impl.MapService) MapServiceContext(com.hazelcast.map.impl.MapServiceContext)

Example 3 with ORDERED

use of java.util.Spliterator.ORDERED in project hazelcast by hazelcast.

the class CsvReadFileFnProvider method createReadFileFn.

@SuppressWarnings("unchecked")
@Nonnull
@Override
public <T> FunctionEx<Path, Stream<T>> createReadFileFn(@Nonnull FileFormat<T> format) {
    CsvFileFormat<T> csvFileFormat = (CsvFileFormat<T>) format;
    // Format is not Serializable
    Class<?> formatClazz = csvFileFormat.clazz();
    return path -> {
        FileInputStream fis = new FileInputStream(path.toFile());
        MappingIterator<T> iterator;
        Function<T, T> projection = identity();
        if (formatClazz == String[].class) {
            ObjectReader reader = new CsvMapper().enable(Feature.WRAP_AS_ARRAY).readerFor(String[].class).with(CsvSchema.emptySchema().withSkipFirstDataRow(false));
            iterator = reader.readValues(fis);
            if (!iterator.hasNext()) {
                throw new JetException("Header row missing in " + path);
            }
            String[] header = (String[]) iterator.next();
            List<String> fieldNames = csvFileFormat.fieldNames();
            if (fieldNames != null) {
                projection = (Function<T, T>) createFieldProjection(header, fieldNames);
            }
        } else {
            iterator = new CsvMapper().readerFor(formatClazz).withoutFeatures(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).with(CsvSchema.emptySchema().withHeader()).readValues(fis);
        }
        return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, ORDERED), false).map(projection).onClose(() -> uncheckRun(fis::close));
    };
}
Also used : FunctionEx(com.hazelcast.function.FunctionEx) Util.uncheckRun(com.hazelcast.jet.impl.util.Util.uncheckRun) Spliterators(java.util.Spliterators) MappingIterator(com.fasterxml.jackson.databind.MappingIterator) CsvMapper(com.fasterxml.jackson.dataformat.csv.CsvMapper) CsvSchema(com.fasterxml.jackson.dataformat.csv.CsvSchema) ORDERED(java.util.Spliterator.ORDERED) FileInputStream(java.io.FileInputStream) Function(java.util.function.Function) DeserializationFeature(com.fasterxml.jackson.databind.DeserializationFeature) ObjectReader(com.fasterxml.jackson.databind.ObjectReader) JetException(com.hazelcast.jet.JetException) CsvFileFormat(com.hazelcast.jet.pipeline.file.CsvFileFormat) Feature(com.fasterxml.jackson.dataformat.csv.CsvParser.Feature) FileFormat(com.hazelcast.jet.pipeline.file.FileFormat) List(java.util.List) Stream(java.util.stream.Stream) Util.createFieldProjection(com.hazelcast.jet.impl.util.Util.createFieldProjection) ReadFileFnProvider(com.hazelcast.jet.pipeline.file.impl.ReadFileFnProvider) Function.identity(java.util.function.Function.identity) StreamSupport(java.util.stream.StreamSupport) Nonnull(javax.annotation.Nonnull) Path(java.nio.file.Path) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) CsvMapper(com.fasterxml.jackson.dataformat.csv.CsvMapper) JetException(com.hazelcast.jet.JetException) FileInputStream(java.io.FileInputStream) Function(java.util.function.Function) MappingIterator(com.fasterxml.jackson.databind.MappingIterator) ObjectReader(com.fasterxml.jackson.databind.ObjectReader) List(java.util.List) CsvFileFormat(com.hazelcast.jet.pipeline.file.CsvFileFormat) Nonnull(javax.annotation.Nonnull)

Example 4 with ORDERED

use of java.util.Spliterator.ORDERED in project sql-boot by sql-boot.

the class JdbcSqlQuery method getMapStream.

private Stream<Map<String, Object>> getMapStream(final String sqlText) {
    logger.info(sqlText);
    final SqlRowSet rowSet = new JdbcTemplate(dataSource).queryForRowSet(sqlText);
    Iterator<Map<String, Object>> iterator = new Iterator<Map<String, Object>>() {

        @Override
        public boolean hasNext() {
            return rowSet.next();
        }

        @Override
        public Map<String, Object> next() {
            return stream(rowSet.getMetaData().getColumnNames()).map(v -> {
                Object object = rowSet.getObject(v);
                if (object instanceof SerialClob) {
                    try {
                        return new SimpleEntry<>(v, (Object) ((SerialClob) object).getSubString(1, (int) ((SerialClob) object).length()));
                    } catch (SerialException e) {
                        e.printStackTrace();
                    }
                }
                return new SimpleEntry<>(v, object);
            }).collect(toMap(k -> k.getKey(), /*.toLowerCase()*/
            v -> ofNullable(v.getValue()).orElse(nullAlias), (a, b) -> a, LinkedHashMap::new));
        }
    };
    return stream(spliteratorUnknownSize(iterator, ORDERED), false);
}
Also used : SqlRowSet(org.springframework.jdbc.support.rowset.SqlRowSet) Spliterators.spliteratorUnknownSize(java.util.Spliterators.spliteratorUnknownSize) Connection(java.sql.Connection) Iterator(java.util.Iterator) SerialClob(javax.sql.rowset.serial.SerialClob) Optional.ofNullable(java.util.Optional.ofNullable) TemplateGenerator(com.github.mgramin.sqlboot.template.generator.TemplateGenerator) SqlQuery(com.github.mgramin.sqlboot.sql.SqlQuery) ORDERED(java.util.Spliterator.ORDERED) BootException(com.github.mgramin.sqlboot.exceptions.BootException) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) LinkedHashMap(java.util.LinkedHashMap) SelectStatementParser(com.github.mgramin.sqlboot.sql.parser.SelectStatementParser) Logger(org.apache.log4j.Logger) SQLException(java.sql.SQLException) Stream(java.util.stream.Stream) Collectors.toMap(java.util.stream.Collectors.toMap) StreamSupport.stream(java.util.stream.StreamSupport.stream) SerialException(javax.sql.rowset.serial.SerialException) Map(java.util.Map) DataSource(javax.sql.DataSource) SqlRowSet(org.springframework.jdbc.support.rowset.SqlRowSet) SimpleEntry(java.util.AbstractMap.SimpleEntry) Arrays.stream(java.util.Arrays.stream) SerialException(javax.sql.rowset.serial.SerialException) Iterator(java.util.Iterator) SerialClob(javax.sql.rowset.serial.SerialClob) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) LinkedHashMap(java.util.LinkedHashMap) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map)

Aggregations

ORDERED (java.util.Spliterator.ORDERED)4 Iterator (java.util.Iterator)3 Spliterators.spliteratorUnknownSize (java.util.Spliterators.spliteratorUnknownSize)3 StreamSupport.stream (java.util.stream.StreamSupport.stream)3 Data (com.hazelcast.internal.serialization.Data)2 InternalSerializationService (com.hazelcast.internal.serialization.InternalSerializationService)2 InternalGenericRecord (com.hazelcast.internal.serialization.impl.InternalGenericRecord)2 Util.entry (com.hazelcast.jet.Util.entry)2 Util (com.hazelcast.jet.impl.util.Util)2 Util.getNodeEngine (com.hazelcast.jet.impl.util.Util.getNodeEngine)2 SqlTestSupport (com.hazelcast.jet.sql.SqlTestSupport)2 JAVA_FORMAT (com.hazelcast.jet.sql.impl.connector.SqlConnector.JAVA_FORMAT)2 OPTION_KEY_CLASS (com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_CLASS)2 OPTION_KEY_FORMAT (com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_KEY_FORMAT)2 OPTION_VALUE_FORMAT (com.hazelcast.jet.sql.impl.connector.SqlConnector.OPTION_VALUE_FORMAT)2 TestAllTypesSqlConnector (com.hazelcast.jet.sql.impl.connector.test.TestAllTypesSqlConnector)2 MapService (com.hazelcast.map.impl.MapService)2 MapServiceContext (com.hazelcast.map.impl.MapServiceContext)2 Record (com.hazelcast.map.impl.record.Record)2 GenericRecord (com.hazelcast.nio.serialization.GenericRecord)2