Examples with DataContext org.apache.calcite.DataContext used on opensource projects

Example 21 with DataContext

use of org.apache.calcite.DataContext in project calcite by apache.

the class StdinTableFunction method eval.

public static ScannableTable eval(boolean b) {
    return new ScannableTable() {

        @Override
        public Enumerable<Object[]> scan(DataContext root) {
            final InputStream is = DataContext.Variable.STDIN.get(root);
            return new AbstractEnumerable<Object[]>() {

                final InputStreamReader in = new InputStreamReader(is, StandardCharsets.UTF_8);

                final BufferedReader br = new BufferedReader(in);

                @Override
                public Enumerator<Object[]> enumerator() {
                    return new Enumerator<Object[]>() {

                        @Nullable
                        String line;

                        int i;

                        @Override
                        public Object[] current() {
                            if (line == null) {
                                throw new NoSuchElementException();
                            }
                            return new Object[] { i, line };
                        }

                        @Override
                        public boolean moveNext() {
                            try {
                                line = br.readLine();
                                ++i;
                                return line != null;
                            } catch (IOException e) {
                                throw new RuntimeException(e);
                            }
                        }

                        @Override
                        public void reset() {
                            throw new UnsupportedOperationException();
                        }

                        @Override
                        public void close() {
                            try {
                                br.close();
                            } catch (IOException e) {
                                throw new RuntimeException(e);
                            }
                        }
                    };
                }
            };
        }

        @Override
        public RelDataType getRowType(RelDataTypeFactory typeFactory) {
            return typeFactory.builder().add("ordinal", SqlTypeName.INTEGER).add("line", SqlTypeName.VARCHAR).build();
        }

        @Override
        public Statistic getStatistic() {
            return Statistics.of(1000d, ImmutableList.of(ImmutableBitSet.of(1)));
        }

        @Override
        public Schema.TableType getJdbcTableType() {
            return Schema.TableType.TABLE;
        }

        @Override
        public boolean isRolledUp(String column) {
            return false;
        }

        @Override
        public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call, @Nullable SqlNode parent, @Nullable CalciteConnectionConfig config) {
            return true;
        }
    };
}

Example 22 with DataContext

use of org.apache.calcite.DataContext in project calcite by apache.

the class FilesTableFunction method eval.

/**
 * Evaluates the function.
 *
 * @param path Directory in which to start the search. Typically '.'
 * @return Table that can be inspected, planned, and evaluated
 */
public static ScannableTable eval(final String path) {
    return new ScannableTable() {

        @Override
        public RelDataType getRowType(RelDataTypeFactory typeFactory) {
            return typeFactory.builder().add("access_time", // %A@ sec since epoch
            SqlTypeName.TIMESTAMP).add("block_count", // %b in 512B blocks
            SqlTypeName.INTEGER).add("change_time", // %C@ sec since epoch
            SqlTypeName.TIMESTAMP).add("depth", // %d depth in directory tree
            SqlTypeName.INTEGER).add("device", // %D device number
            SqlTypeName.INTEGER).add("file_name", // %f file name, sans dirs
            SqlTypeName.VARCHAR).add("fstype", // %F file system type
            SqlTypeName.VARCHAR).add("gname", // %g group name
            SqlTypeName.VARCHAR).add("gid", // %G numeric group id
            SqlTypeName.INTEGER).add("dir_name", // %h leading dirs
            SqlTypeName.VARCHAR).add("inode", // %i inode number
            SqlTypeName.BIGINT).add("link", // %l object of sym link
            SqlTypeName.VARCHAR).add("perm", SqlTypeName.CHAR, // %#m permission octal
            4).add("hard", // %n number of hard links
            SqlTypeName.INTEGER).add("path", // %P file's name
            SqlTypeName.VARCHAR).add("size", // %s file's size in bytes
            SqlTypeName.BIGINT).add("mod_time", // %T@ seconds since epoch
            SqlTypeName.TIMESTAMP).add("user", // %u user name
            SqlTypeName.VARCHAR).add("uid", // %U numeric user id
            SqlTypeName.INTEGER).add("type", SqlTypeName.CHAR, // %Y file type
            1).build();
        // Fields in Linux find that are currently ignored:
        // %y file type (not following sym links)
        // %k block count in 1KB blocks
        // %p file name (including argument)
        }

        private Enumerable<String> sourceLinux() {
            final String[] args = { "find", path, "-printf", "" + // access_time
            "%A@\\0" + // block_count
            "%b\\0" + // change_time
            "%C@\\0" + // depth
            "%d\\0" + // device
            "%D\\0" + // file_name
            "%f\\0" + // fstype
            "%F\\0" + // gname
            "%g\\0" + // gid
            "%G\\0" + // dir_name
            "%h\\0" + // inode
            "%i\\0" + // link
            "%l\\0" + // perm
            "%#m\\0" + // hard
            "%n\\0" + // path
            "%P\\0" + // size
            "%s\\0" + // mod_time
            "%T@\\0" + // user
            "%u\\0" + // uid
            "%U\\0" + // type
            "%Y\\0" };
            return Processes.processLines('\0', args);
        }

        private Enumerable<String> sourceMacOs() {
            if (path.contains("'")) {
                // no injection monkey business
                throw new IllegalArgumentException();
            }
            final String[] args = { "/bin/sh", "-c", "find '" + path + "' | xargs stat -f " + // access_time
            "%a%n" + // block_count
            "%b%n" + // change_time
            "%c%n" + // depth: not supported by macOS stat
            "0%n" + // device: we only use the high part of "H,L" device
            "%Hd%n" + // filename: not supported by macOS stat
            "filename%n" + // fstype: not supported by macOS stat
            "fstype%n" + // gname
            "%Sg%n" + // gid
            "%g%n" + // dir_name: not supported by macOS stat
            "dir_name%n" + // inode
            "%i%n" + // link
            "%Y%n" + // perm
            "%Lp%n" + // hard
            "%l%n" + // path
            "%SN%n" + // size
            "%z%n" + // mod_time
            "%m%n" + // user
            "%Su%n" + // uid
            "%u%n" + // type
            "%LT%n" };
            return Processes.processLines('\n', args);
        }

        @Override
        public Enumerable<@Nullable Object[]> scan(DataContext root) {
            JavaTypeFactory typeFactory = root.getTypeFactory();
            final RelDataType rowType = getRowType(typeFactory);
            final List<String> fieldNames = ImmutableList.copyOf(rowType.getFieldNames());
            final String osName = System.getProperty("os.name");
            final String osVersion = System.getProperty("os.version");
            Util.discard(osVersion);
            final Enumerable<String> enumerable;
            switch(osName) {
                case // tested on version 10.12.5
                "Mac OS X":
                    enumerable = sourceMacOs();
                    break;
                default:
                    enumerable = sourceLinux();
            }
            return new AbstractEnumerable<@Nullable Object[]>() {

                @Override
                public Enumerator<@Nullable Object[]> enumerator() {
                    final Enumerator<String> e = enumerable.enumerator();
                    return new Enumerator<@Nullable Object[]>() {

                        @Nullable
                        Object @Nullable [] current;

                        @Override
                        public Object[] current() {
                            return requireNonNull(current, "current");
                        }

                        @Override
                        public boolean moveNext() {
                            current = new Object[fieldNames.size()];
                            for (int i = 0; i < current.length; i++) {
                                if (!e.moveNext()) {
                                    return false;
                                }
                                final String v = e.current();
                                try {
                                    current[i] = field(fieldNames.get(i), v);
                                } catch (RuntimeException e) {
                                    throw new RuntimeException("while parsing value [" + v + "] of field [" + fieldNames.get(i) + "] in line [" + Arrays.toString(current) + "]", e);
                                }
                            }
                            switch(osName) {
                                case "Mac OS X":
                                    // Strip leading "./"
                                    String path = (String) current[14];
                                    if (".".equals(path)) {
                                        current[14] = path = "";
                                        // depth
                                        current[3] = 0;
                                    } else if (path.startsWith("./")) {
                                        current[14] = path = path.substring(2);
                                        // depth
                                        current[3] = count(path, '/') + 1;
                                    } else {
                                        // depth
                                        current[3] = count(path, '/');
                                    }
                                    final int slash = path.lastIndexOf('/');
                                    if (slash >= 0) {
                                        // filename
                                        current[5] = path.substring(slash + 1);
                                        // dir_name
                                        current[9] = path.substring(0, slash);
                                    } else {
                                        // filename
                                        current[5] = path;
                                        // dir_name
                                        current[9] = "";
                                    }
                                    // Make type values more like those on Linux
                                    final String type = (String) current[19];
                                    current[19] = "/".equals(type) ? "d" : "".equals(type) || "*".equals(type) ? "f" : "@".equals(type) ? "l" : type;
                                    break;
                                default:
                                    break;
                            }
                            return true;
                        }

                        private int count(String s, char c) {
                            int n = 0;
                            for (int i = 0, len = s.length(); i < len; i++) {
                                if (s.charAt(i) == c) {
                                    ++n;
                                }
                            }
                            return n;
                        }

                        @Override
                        public void reset() {
                            throw new UnsupportedOperationException();
                        }

                        @Override
                        public void close() {
                            e.close();
                        }

                        private Object field(String field, String value) {
                            switch(field) {
                                case "block_count":
                                case "depth":
                                case "device":
                                case "gid":
                                case "uid":
                                case "hard":
                                    return Integer.valueOf(value);
                                case "inode":
                                case "size":
                                    return Long.valueOf(value);
                                case "access_time":
                                case "change_time":
                                case "mod_time":
                                    return new BigDecimal(value).multiply(THOUSAND).longValue();
                                default:
                                    return value;
                            }
                        }
                    };
                }
            };
        }

        @Override
        public Statistic getStatistic() {
            return Statistics.of(1000d, ImmutableList.of(ImmutableBitSet.of(1)));
        }

        @Override
        public Schema.TableType getJdbcTableType() {
            return Schema.TableType.TABLE;
        }

        @Override
        public boolean isRolledUp(String column) {
            return false;
        }

        @Override
        public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call, @Nullable SqlNode parent, @Nullable CalciteConnectionConfig config) {
            return true;
        }
    };
}

Example 23 with DataContext

use of org.apache.calcite.DataContext in project druid by apache.

the class SystemSchemaTest method testServerSegmentsTable.

@Test
public void testServerSegmentsTable() {
    SystemSchema.ServerSegmentsTable serverSegmentsTable = EasyMock.createMockBuilder(SystemSchema.ServerSegmentsTable.class).withConstructor(serverView, authMapper).createMock();
    EasyMock.replay(serverSegmentsTable);
    EasyMock.expect(serverView.getDruidServers()).andReturn(immutableDruidServers).once();
    EasyMock.replay(serverView);
    DataContext dataContext = createDataContext(Users.SUPER);
    // server_segments table is the join of servers and segments table
    // it will have 5 rows as follows
    // localhost:0000 |  test1_2010-01-01T00:00:00.000Z_2011-01-01T00:00:00.000Z_version1(segment1)
    // localhost:0000 |  test2_2011-01-01T00:00:00.000Z_2012-01-01T00:00:00.000Z_version2(segment2)
    // server2:1234   |  test3_2012-01-01T00:00:00.000Z_2013-01-01T00:00:00.000Z_version3(segment3)
    // server2:1234   |  test4_2017-01-01T00:00:00.000Z_2018-01-01T00:00:00.000Z_version4(segment4)
    // server2:1234   |  test5_2017-01-01T00:00:00.000Z_2018-01-01T00:00:00.000Z_version5(segment5)
    final List<Object[]> rows = serverSegmentsTable.scan(dataContext).toList();
    Assert.assertEquals(5, rows.size());
    Object[] row0 = rows.get(0);
    Assert.assertEquals("localhost:0000", row0[0]);
    Assert.assertEquals("test1_2010-01-01T00:00:00.000Z_2011-01-01T00:00:00.000Z_version1", row0[1].toString());
    Object[] row1 = rows.get(1);
    Assert.assertEquals("localhost:0000", row1[0]);
    Assert.assertEquals("test2_2011-01-01T00:00:00.000Z_2012-01-01T00:00:00.000Z_version2", row1[1].toString());
    Object[] row2 = rows.get(2);
    Assert.assertEquals("server2:1234", row2[0]);
    Assert.assertEquals("test3_2012-01-01T00:00:00.000Z_2013-01-01T00:00:00.000Z_version3_2", row2[1].toString());
    Object[] row3 = rows.get(3);
    Assert.assertEquals("server2:1234", row3[0]);
    Assert.assertEquals("test4_2014-01-01T00:00:00.000Z_2015-01-01T00:00:00.000Z_version4", row3[1].toString());
    Object[] row4 = rows.get(4);
    Assert.assertEquals("server2:1234", row4[0]);
    Assert.assertEquals("test5_2015-01-01T00:00:00.000Z_2016-01-01T00:00:00.000Z_version5", row4[1].toString());
    // Verify value types.
    verifyTypes(rows, SystemSchema.SERVER_SEGMENTS_SIGNATURE);
}

Example 24 with DataContext

use of org.apache.calcite.DataContext in project druid by apache.

the class SystemSchemaTest method testTasksTable.

@Test
public void testTasksTable() throws Exception {
    SystemSchema.TasksTable tasksTable = EasyMock.createMockBuilder(SystemSchema.TasksTable.class).withConstructor(client, mapper, authMapper).createMock();
    EasyMock.replay(tasksTable);
    EasyMock.expect(client.makeRequest(HttpMethod.GET, "/druid/indexer/v1/tasks")).andReturn(request).anyTimes();
    HttpResponse httpResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    InputStreamFullResponseHolder responseHolder = new InputStreamFullResponseHolder(httpResp);
    EasyMock.expect(client.go(EasyMock.eq(request), EasyMock.anyObject(InputStreamFullResponseHandler.class))).andReturn(responseHolder).once();
    EasyMock.expect(request.getUrl()).andReturn(new URL("http://test-host:1234/druid/indexer/v1/tasks")).anyTimes();
    String json = "[{\n" + "\t\"id\": \"index_wikipedia_2018-09-20T22:33:44.911Z\",\n" + "\t\"groupId\": \"group_index_wikipedia_2018-09-20T22:33:44.911Z\",\n" + "\t\"type\": \"index\",\n" + "\t\"createdTime\": \"2018-09-20T22:33:44.922Z\",\n" + "\t\"queueInsertionTime\": \"1970-01-01T00:00:00.000Z\",\n" + "\t\"statusCode\": \"FAILED\",\n" + "\t\"runnerStatusCode\": \"NONE\",\n" + "\t\"duration\": -1,\n" + "\t\"location\": {\n" + "\t\t\"host\": \"testHost\",\n" + "\t\t\"port\": 1234,\n" + "\t\t\"tlsPort\": -1\n" + "\t},\n" + "\t\"dataSource\": \"wikipedia\",\n" + "\t\"errorMsg\": null\n" + "}, {\n" + "\t\"id\": \"index_wikipedia_2018-09-21T18:38:47.773Z\",\n" + "\t\"groupId\": \"group_index_wikipedia_2018-09-21T18:38:47.773Z\",\n" + "\t\"type\": \"index\",\n" + "\t\"createdTime\": \"2018-09-21T18:38:47.873Z\",\n" + "\t\"queueInsertionTime\": \"2018-09-21T18:38:47.910Z\",\n" + "\t\"statusCode\": \"RUNNING\",\n" + "\t\"runnerStatusCode\": \"RUNNING\",\n" + "\t\"duration\": null,\n" + "\t\"location\": {\n" + "\t\t\"host\": \"192.168.1.6\",\n" + "\t\t\"port\": 8100,\n" + "\t\t\"tlsPort\": -1\n" + "\t},\n" + "\t\"dataSource\": \"wikipedia\",\n" + "\t\"errorMsg\": null\n" + "}]";
    byte[] bytesToWrite = json.getBytes(StandardCharsets.UTF_8);
    responseHolder.addChunk(bytesToWrite);
    responseHolder.done();
    EasyMock.replay(client, request, responseHandler);
    DataContext dataContext = createDataContext(Users.SUPER);
    final List<Object[]> rows = tasksTable.scan(dataContext).toList();
    Object[] row0 = rows.get(0);
    Assert.assertEquals("index_wikipedia_2018-09-20T22:33:44.911Z", row0[0].toString());
    Assert.assertEquals("group_index_wikipedia_2018-09-20T22:33:44.911Z", row0[1].toString());
    Assert.assertEquals("index", row0[2].toString());
    Assert.assertEquals("wikipedia", row0[3].toString());
    Assert.assertEquals("2018-09-20T22:33:44.922Z", row0[4].toString());
    Assert.assertEquals("1970-01-01T00:00:00.000Z", row0[5].toString());
    Assert.assertEquals("FAILED", row0[6].toString());
    Assert.assertEquals("NONE", row0[7].toString());
    Assert.assertEquals(-1L, row0[8]);
    Assert.assertEquals("testHost:1234", row0[9]);
    Assert.assertEquals("testHost", row0[10]);
    Assert.assertEquals(1234L, row0[11]);
    Assert.assertEquals(-1L, row0[12]);
    Assert.assertEquals(null, row0[13]);
    Object[] row1 = rows.get(1);
    Assert.assertEquals("index_wikipedia_2018-09-21T18:38:47.773Z", row1[0].toString());
    Assert.assertEquals("group_index_wikipedia_2018-09-21T18:38:47.773Z", row1[1].toString());
    Assert.assertEquals("index", row1[2].toString());
    Assert.assertEquals("wikipedia", row1[3].toString());
    Assert.assertEquals("2018-09-21T18:38:47.873Z", row1[4].toString());
    Assert.assertEquals("2018-09-21T18:38:47.910Z", row1[5].toString());
    Assert.assertEquals("RUNNING", row1[6].toString());
    Assert.assertEquals("RUNNING", row1[7].toString());
    Assert.assertEquals(0L, row1[8]);
    Assert.assertEquals("192.168.1.6:8100", row1[9]);
    Assert.assertEquals("192.168.1.6", row1[10]);
    Assert.assertEquals(8100L, row1[11]);
    Assert.assertEquals(-1L, row1[12]);
    Assert.assertEquals(null, row1[13]);
    // Verify value types.
    verifyTypes(rows, SystemSchema.TASKS_SIGNATURE);
}

Example 25 with DataContext

use of org.apache.calcite.DataContext in project plc4x by apache.

the class Plc4xBaseTable method scan.

/**
 * if tableCutoff is positive, then the row gets limited to that.
 */
public Enumerable<Object[]> scan(DataContext root) {
    return new AbstractEnumerable<Object[]>() {

        @Override
        public Enumerator<Object[]> enumerator() {
            return new Enumerator<Object[]>() {

                private final AtomicLong counter = new AtomicLong(0);

                @Override
                public Object[] current() {
                    List<Object> objects = new ArrayList<>(Arrays.asList(new Timestamp(current.timestamp.toEpochMilli()), current.source));
                    List<Object> objects2 = names.stream().map(name -> current.values.get(name)).collect(Collectors.toList());
                    objects.addAll(objects2);
                    return objects.toArray();
                }

                @Override
                public boolean moveNext() {
                    try {
                        current = queue.take();
                        // If stream, simply return
                        if (tableCutoff <= 0L) {
                            return true;
                        }
                        // If table, return if below cutoff
                        return counter.getAndIncrement() < tableCutoff;
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    }
                    return false;
                }

                @Override
                public void reset() {
                    counter.set(0);
                }

                @Override
                public void close() {
                // Unimplemented
                }
            };
        }
    };
}