Examples with Enumerator org.apache.calcite.linq4j.Enumerator used on opensource projects

Example 1 with Enumerator

use of org.apache.calcite.linq4j.Enumerator in project calcite by apache.

the class EnumerableCalc method implement.

public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
    final JavaTypeFactory typeFactory = implementor.getTypeFactory();
    final BlockBuilder builder = new BlockBuilder();
    final EnumerableRel child = (EnumerableRel) getInput();
    final Result result = implementor.visitChild(this, 0, child, pref);
    final PhysType physType = PhysTypeImpl.of(typeFactory, getRowType(), pref.prefer(result.format));
    // final Enumerable<Employee> inputEnumerable = <<child adapter>>;
    // return new Enumerable<IntString>() {
    // Enumerator<IntString> enumerator() {
    // return new Enumerator<IntString>() {
    // public void reset() {
    // ...
    Type outputJavaType = physType.getJavaRowType();
    final Type enumeratorType = Types.of(Enumerator.class, outputJavaType);
    Type inputJavaType = result.physType.getJavaRowType();
    ParameterExpression inputEnumerator = Expressions.parameter(Types.of(Enumerator.class, inputJavaType), "inputEnumerator");
    Expression input = RexToLixTranslator.convert(Expressions.call(inputEnumerator, BuiltInMethod.ENUMERATOR_CURRENT.method), inputJavaType);
    final RexBuilder rexBuilder = getCluster().getRexBuilder();
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RelOptPredicateList predicates = mq.getPulledUpPredicates(child);
    final RexSimplify simplify = new RexSimplify(rexBuilder, predicates, false, RexUtil.EXECUTOR);
    final RexProgram program = this.program.normalize(rexBuilder, simplify);
    BlockStatement moveNextBody;
    if (program.getCondition() == null) {
        moveNextBody = Blocks.toFunctionBlock(Expressions.call(inputEnumerator, BuiltInMethod.ENUMERATOR_MOVE_NEXT.method));
    } else {
        final BlockBuilder builder2 = new BlockBuilder();
        Expression condition = RexToLixTranslator.translateCondition(program, typeFactory, builder2, new RexToLixTranslator.InputGetterImpl(Collections.singletonList(Pair.of(input, result.physType))), implementor.allCorrelateVariables);
        builder2.add(Expressions.ifThen(condition, Expressions.return_(null, Expressions.constant(true))));
        moveNextBody = Expressions.block(Expressions.while_(Expressions.call(inputEnumerator, BuiltInMethod.ENUMERATOR_MOVE_NEXT.method), builder2.toBlock()), Expressions.return_(null, Expressions.constant(false)));
    }
    final BlockBuilder builder3 = new BlockBuilder();
    List<Expression> expressions = RexToLixTranslator.translateProjects(program, typeFactory, builder3, physType, DataContext.ROOT, new RexToLixTranslator.InputGetterImpl(Collections.singletonList(Pair.of(input, result.physType))), implementor.allCorrelateVariables);
    builder3.add(Expressions.return_(null, physType.record(expressions)));
    BlockStatement currentBody = builder3.toBlock();
    final Expression inputEnumerable = builder.append("inputEnumerable", result.block, false);
    final Expression body = Expressions.new_(enumeratorType, NO_EXPRS, Expressions.list(Expressions.fieldDecl(Modifier.PUBLIC | Modifier.FINAL, inputEnumerator, Expressions.call(inputEnumerable, BuiltInMethod.ENUMERABLE_ENUMERATOR.method)), EnumUtils.overridingMethodDecl(BuiltInMethod.ENUMERATOR_RESET.method, NO_PARAMS, Blocks.toFunctionBlock(Expressions.call(inputEnumerator, BuiltInMethod.ENUMERATOR_RESET.method))), EnumUtils.overridingMethodDecl(BuiltInMethod.ENUMERATOR_MOVE_NEXT.method, NO_PARAMS, moveNextBody), EnumUtils.overridingMethodDecl(BuiltInMethod.ENUMERATOR_CLOSE.method, NO_PARAMS, Blocks.toFunctionBlock(Expressions.call(inputEnumerator, BuiltInMethod.ENUMERATOR_CLOSE.method))), Expressions.methodDecl(Modifier.PUBLIC, BRIDGE_METHODS ? Object.class : outputJavaType, "current", NO_PARAMS, currentBody)));
    builder.add(Expressions.return_(null, Expressions.new_(BuiltInMethod.ABSTRACT_ENUMERABLE_CTOR.constructor, // Collections.singletonList(inputRowType),
    NO_EXPRS, ImmutableList.<MemberDeclaration>of(Expressions.methodDecl(Modifier.PUBLIC, enumeratorType, BuiltInMethod.ENUMERABLE_ENUMERATOR.method.getName(), NO_PARAMS, Blocks.toFunctionBlock(body))))));
    return implementor.result(physType, builder.toBlock());
}

Example 2 with Enumerator

use of org.apache.calcite.linq4j.Enumerator in project calcite by apache.

the class FilesTableFunction method eval.

/**
 * Evaluates the function.
 *
 * @param path Directory in which to start the search. Typically '.'
 * @return Table that can be inspected, planned, and evaluated
 */
public static ScannableTable eval(final String path) {
    return new ScannableTable() {

        public RelDataType getRowType(RelDataTypeFactory typeFactory) {
            return typeFactory.builder().add("access_time", // %A@ sec since epoch
            SqlTypeName.TIMESTAMP).add("block_count", // %b in 512B blocks
            SqlTypeName.INTEGER).add("change_time", // %C@ sec since epoch
            SqlTypeName.TIMESTAMP).add("depth", // %d depth in directory tree
            SqlTypeName.INTEGER).add("device", // %D device number
            SqlTypeName.INTEGER).add("file_name", // %f file name, sans dirs
            SqlTypeName.VARCHAR).add("fstype", // %F file system type
            SqlTypeName.VARCHAR).add("gname", // %g group name
            SqlTypeName.VARCHAR).add("gid", // %G numeric group id
            SqlTypeName.INTEGER).add("dir_name", // %h leading dirs
            SqlTypeName.VARCHAR).add("inode", // %i inode number
            SqlTypeName.BIGINT).add("link", // %l object of sym link
            SqlTypeName.VARCHAR).add("perm", SqlTypeName.CHAR, // %#m permission octal
            4).add("hard", // %n number of hard links
            SqlTypeName.INTEGER).add("path", // %P file's name
            SqlTypeName.VARCHAR).add("size", // %s file's size in bytes
            SqlTypeName.BIGINT).add("mod_time", // %T@ seconds since epoch
            SqlTypeName.TIMESTAMP).add("user", // %u user name
            SqlTypeName.VARCHAR).add("uid", // %U numeric user id
            SqlTypeName.INTEGER).add("type", SqlTypeName.CHAR, // %Y file type
            1).build();
        // Fields in Linux find that are currently ignored:
        // %y file type (not following sym links)
        // %k block count in 1KB blocks
        // %p file name (including argument)
        }

        private Enumerable<String> sourceLinux() {
            final String[] args = { "find", path, "-printf", "" + // access_time
            "%A@\\0" + // block_count
            "%b\\0" + // change_time
            "%C@\\0" + // depth
            "%d\\0" + // device
            "%D\\0" + // file_name
            "%f\\0" + // fstype
            "%F\\0" + // gname
            "%g\\0" + // gid
            "%G\\0" + // dir_name
            "%h\\0" + // inode
            "%i\\0" + // link
            "%l\\0" + // perm
            "%#m\\0" + // hard
            "%n\\0" + // path
            "%P\\0" + // size
            "%s\\0" + // mod_time
            "%T@\\0" + // user
            "%u\\0" + // uid
            "%U\\0" + // type
            "%Y\\0" };
            return Processes.processLines('\0', args);
        }

        private Enumerable<String> sourceMacOs() {
            if (path.contains("'")) {
                // no injection monkey business
                throw new IllegalArgumentException();
            }
            final String[] args = { "/bin/sh", "-c", "find '" + path + "' | xargs stat -f " + // access_time
            "%a%n" + // block_count
            "%b%n" + // change_time
            "%c%n" + // depth: not supported by macOS stat
            "0%n" + // device: we only use the high part of "H,L" device
            "%Hd%n" + // filename: not supported by macOS stat
            "filename%n" + // fstype: not supported by macOS stat
            "fstype%n" + // gname
            "%Sg%n" + // gid
            "%g%n" + // dir_name: not supported by macOS stat
            "dir_name%n" + // inode
            "%i%n" + // link
            "%Y%n" + // perm
            "%Lp%n" + // hard
            "%l%n" + // path
            "%SN%n" + // size
            "%z%n" + // mod_time
            "%m%n" + // user
            "%Su%n" + // uid
            "%u%n" + // type
            "%LT%n" };
            return Processes.processLines('\n', args);
        }

        public Enumerable<Object[]> scan(DataContext root) {
            final RelDataType rowType = getRowType(root.getTypeFactory());
            final List<String> fieldNames = ImmutableList.copyOf(rowType.getFieldNames());
            final String osName = System.getProperty("os.name");
            final String osVersion = System.getProperty("os.version");
            Util.discard(osVersion);
            final Enumerable<String> enumerable;
            switch(osName) {
                case // tested on version 10.12.5
                "Mac OS X":
                    enumerable = sourceMacOs();
                    break;
                default:
                    enumerable = sourceLinux();
            }
            return new AbstractEnumerable<Object[]>() {

                public Enumerator<Object[]> enumerator() {
                    final Enumerator<String> e = enumerable.enumerator();
                    return new Enumerator<Object[]>() {

                        Object[] current;

                        public Object[] current() {
                            return current;
                        }

                        public boolean moveNext() {
                            current = new Object[fieldNames.size()];
                            for (int i = 0; i < current.length; i++) {
                                if (!e.moveNext()) {
                                    return false;
                                }
                                final String v = e.current();
                                try {
                                    current[i] = field(fieldNames.get(i), v);
                                } catch (RuntimeException e) {
                                    throw new RuntimeException("while parsing value [" + v + "] of field [" + fieldNames.get(i) + "] in line [" + Arrays.toString(current) + "]", e);
                                }
                            }
                            switch(osName) {
                                case "Mac OS X":
                                    // Strip leading "./"
                                    String path = (String) current[14];
                                    if (path.equals(".")) {
                                        current[14] = path = "";
                                        // depth
                                        current[3] = 0;
                                    } else if (path.startsWith("./")) {
                                        current[14] = path = path.substring(2);
                                        // depth
                                        current[3] = count(path, '/') + 1;
                                    } else {
                                        // depth
                                        current[3] = count(path, '/');
                                    }
                                    final int slash = path.lastIndexOf('/');
                                    if (slash >= 0) {
                                        // filename
                                        current[5] = path.substring(slash + 1);
                                        // dir_name
                                        current[9] = path.substring(0, slash);
                                    } else {
                                        // filename
                                        current[5] = path;
                                        // dir_name
                                        current[9] = "";
                                    }
                                    // Make type values more like those on Linux
                                    final String type = (String) current[19];
                                    current[19] = type.equals("/") ? "d" : type.equals("") || type.equals("*") ? "f" : type.equals("@") ? "l" : type;
                            }
                            return true;
                        }

                        private int count(String s, char c) {
                            int n = 0;
                            for (int i = 0, len = s.length(); i < len; i++) {
                                if (s.charAt(i) == c) {
                                    ++n;
                                }
                            }
                            return n;
                        }

                        public void reset() {
                            throw new UnsupportedOperationException();
                        }

                        public void close() {
                            e.close();
                        }

                        private Object field(String field, String value) {
                            switch(field) {
                                case "block_count":
                                case "depth":
                                case "device":
                                case "gid":
                                case "uid":
                                case "hard":
                                    return Integer.valueOf(value);
                                case "inode":
                                case "size":
                                    return Long.valueOf(value);
                                case "access_time":
                                case "change_time":
                                case "mod_time":
                                    return new BigDecimal(value).multiply(THOUSAND).longValue();
                                default:
                                    return value;
                            }
                        }
                    };
                }
            };
        }

        public Statistic getStatistic() {
            return Statistics.of(1000d, ImmutableList.of(ImmutableBitSet.of(1)));
        }

        public Schema.TableType getJdbcTableType() {
            return Schema.TableType.TABLE;
        }

        public boolean isRolledUp(String column) {
            return false;
        }

        public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call, SqlNode parent, CalciteConnectionConfig config) {
            return true;
        }
    };
}

Example 3 with Enumerator

use of org.apache.calcite.linq4j.Enumerator in project calcite by apache.

the class StdinTableFunction method eval.

public static ScannableTable eval(boolean b) {
    return new ScannableTable() {

        public Enumerable<Object[]> scan(DataContext root) {
            final InputStream is = DataContext.Variable.STDIN.get(root);
            return new AbstractEnumerable<Object[]>() {

                final InputStreamReader in = new InputStreamReader(is, StandardCharsets.UTF_8);

                final BufferedReader br = new BufferedReader(in);

                public Enumerator<Object[]> enumerator() {
                    return new Enumerator<Object[]>() {

                        String line;

                        int i;

                        public Object[] current() {
                            if (line == null) {
                                throw new NoSuchElementException();
                            }
                            return new Object[] { i, line };
                        }

                        public boolean moveNext() {
                            try {
                                line = br.readLine();
                                ++i;
                                return line != null;
                            } catch (IOException e) {
                                throw new RuntimeException(e);
                            }
                        }

                        public void reset() {
                            throw new UnsupportedOperationException();
                        }

                        public void close() {
                            try {
                                br.close();
                            } catch (IOException e) {
                                throw new RuntimeException(e);
                            }
                        }
                    };
                }
            };
        }

        public RelDataType getRowType(RelDataTypeFactory typeFactory) {
            return typeFactory.builder().add("ordinal", SqlTypeName.INTEGER).add("line", SqlTypeName.VARCHAR).build();
        }

        public Statistic getStatistic() {
            return Statistics.of(1000d, ImmutableList.of(ImmutableBitSet.of(1)));
        }

        public Schema.TableType getJdbcTableType() {
            return Schema.TableType.TABLE;
        }

        public boolean isRolledUp(String column) {
            return false;
        }

        public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call, SqlNode parent, CalciteConnectionConfig config) {
            return true;
        }
    };
}

Example 4 with Enumerator

use of org.apache.calcite.linq4j.Enumerator in project druid by druid-io.

the class DruidPlanner method planWithBindableConvention.

/**
 * Construct a {@link PlannerResult} for a fall-back 'bindable' rel, for things that are not directly translatable
 * to native Druid queries such as system tables and just a general purpose (but definitely not optimized) fall-back.
 *
 * See {@link #planWithDruidConvention} which will handle things which are directly translatable
 * to native Druid queries.
 */
private PlannerResult planWithBindableConvention(final RelRoot root, @Nullable final SqlExplain explain) throws RelConversionException {
    BindableRel bindableRel = (BindableRel) planner.transform(Rules.BINDABLE_CONVENTION_RULES, planner.getEmptyTraitSet().replace(BindableConvention.INSTANCE).plus(root.collation), root.rel);
    if (!root.isRefTrivial()) {
        // Add a projection on top to accommodate root.fields.
        final List<RexNode> projects = new ArrayList<>();
        final RexBuilder rexBuilder = bindableRel.getCluster().getRexBuilder();
        for (int field : Pair.left(root.fields)) {
            projects.add(rexBuilder.makeInputRef(bindableRel, field));
        }
        bindableRel = new Bindables.BindableProject(bindableRel.getCluster(), bindableRel.getTraitSet(), bindableRel, projects, root.validatedRowType);
    }
    if (explain != null) {
        return planExplanation(bindableRel, explain, false);
    } else {
        final BindableRel theRel = bindableRel;
        final DataContext dataContext = plannerContext.createDataContext((JavaTypeFactory) planner.getTypeFactory(), plannerContext.getParameters());
        final Supplier<Sequence<Object[]>> resultsSupplier = () -> {
            final Enumerable<?> enumerable = theRel.bind(dataContext);
            final Enumerator<?> enumerator = enumerable.enumerator();
            return Sequences.withBaggage(new BaseSequence<>(new BaseSequence.IteratorMaker<Object[], EnumeratorIterator<Object[]>>() {

                @Override
                public EnumeratorIterator<Object[]> make() {
                    return new EnumeratorIterator<>(new Iterator<Object[]>() {

                        @Override
                        public boolean hasNext() {
                            return enumerator.moveNext();
                        }

                        @Override
                        public Object[] next() {
                            return (Object[]) enumerator.current();
                        }
                    });
                }

                @Override
                public void cleanup(EnumeratorIterator<Object[]> iterFromMake) {
                }
            }), enumerator::close);
        };
        return new PlannerResult(resultsSupplier, root.validatedRowType);
    }
}

Example 5 with Enumerator

use of org.apache.calcite.linq4j.Enumerator in project calcite by apache.

the class ScannableTableTest method testPrepared2.

/**
 * Test case for
 * <a href="https://issues.apache.org/jira/browse/CALCITE-1031">[CALCITE-1031]
 * In prepared statement, CsvScannableTable.scan is called twice</a>.
 */
@Test
public void testPrepared2() throws SQLException {
    final Properties properties = new Properties();
    properties.setProperty("caseSensitive", "true");
    try (final Connection connection = DriverManager.getConnection("jdbc:calcite:", properties)) {
        final CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
        final AtomicInteger scanCount = new AtomicInteger();
        final AtomicInteger enumerateCount = new AtomicInteger();
        final Schema schema = new AbstractSchema() {

            @Override
            protected Map<String, Table> getTableMap() {
                return ImmutableMap.<String, Table>of("TENS", new SimpleTable() {

                    private Enumerable<Object[]> superScan(DataContext root) {
                        return super.scan(root);
                    }

                    @Override
                    public Enumerable<Object[]> scan(final DataContext root) {
                        scanCount.incrementAndGet();
                        return new AbstractEnumerable<Object[]>() {

                            public Enumerator<Object[]> enumerator() {
                                enumerateCount.incrementAndGet();
                                return superScan(root).enumerator();
                            }
                        };
                    }
                });
            }
        };
        calciteConnection.getRootSchema().add("TEST", schema);
        final String sql = "select * from \"TEST\".\"TENS\" where \"i\" < ?";
        final PreparedStatement statement = calciteConnection.prepareStatement(sql);
        assertThat(scanCount.get(), is(0));
        assertThat(enumerateCount.get(), is(0));
        // First execute
        statement.setInt(1, 20);
        assertThat(scanCount.get(), is(0));
        ResultSet resultSet = statement.executeQuery();
        assertThat(scanCount.get(), is(1));
        assertThat(enumerateCount.get(), is(1));
        assertThat(resultSet, Matchers.returnsUnordered("i=0", "i=10"));
        assertThat(scanCount.get(), is(1));
        assertThat(enumerateCount.get(), is(1));
        // Second execute
        resultSet = statement.executeQuery();
        assertThat(scanCount.get(), is(2));
        assertThat(resultSet, Matchers.returnsUnordered("i=0", "i=10"));
        assertThat(scanCount.get(), is(2));
        // Third execute
        statement.setInt(1, 30);
        resultSet = statement.executeQuery();
        assertThat(scanCount.get(), is(3));
        assertThat(resultSet, Matchers.returnsUnordered("i=0", "i=10", "i=20"));
        assertThat(scanCount.get(), is(3));
    }
}