Examples with FilteredRows com.google.refine.browsing.FilteredRows used on opensource projects

Example 1 with FilteredRows

use of com.google.refine.browsing.FilteredRows in project OpenRefine by OpenRefine.

the class GetRowsCommand method internalRespond.

protected void internalRespond(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    try {
        Project project = null;
        // This command also supports retrieving rows for an importing job.
        String importingJobID = request.getParameter("importingJobID");
        if (importingJobID != null) {
            long jobID = Long.parseLong(importingJobID);
            ImportingJob job = ImportingManager.getJob(jobID);
            if (job != null) {
                project = job.project;
            }
        }
        if (project == null) {
            project = getProject(request);
        }
        Engine engine = getEngine(request, project);
        String callback = request.getParameter("callback");
        int start = Math.min(project.rows.size(), Math.max(0, getIntegerParameter(request, "start", 0)));
        int limit = Math.min(project.rows.size() - start, Math.max(0, getIntegerParameter(request, "limit", 20)));
        Pool pool = new Pool();
        Properties options = new Properties();
        options.put("project", project);
        options.put("reconCandidateOmitTypes", true);
        options.put("pool", pool);
        response.setCharacterEncoding("UTF-8");
        response.setHeader("Content-Type", callback == null ? "application/json" : "text/javascript");
        PrintWriter writer = response.getWriter();
        if (callback != null) {
            writer.write(callback);
            writer.write("(");
        }
        JSONWriter jsonWriter = new JSONWriter(writer);
        jsonWriter.object();
        RowWritingVisitor rwv = new RowWritingVisitor(start, limit, jsonWriter, options);
        JSONObject sortingJson = null;
        try {
            String json = request.getParameter("sorting");
            sortingJson = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json);
        } catch (JSONException e) {
        }
        if (engine.getMode() == Mode.RowBased) {
            FilteredRows filteredRows = engine.getAllFilteredRows();
            RowVisitor visitor = rwv;
            if (sortingJson != null) {
                SortingRowVisitor srv = new SortingRowVisitor(visitor);
                srv.initializeFromJSON(project, sortingJson);
                if (srv.hasCriteria()) {
                    visitor = srv;
                }
            }
            jsonWriter.key("mode");
            jsonWriter.value("row-based");
            jsonWriter.key("rows");
            jsonWriter.array();
            filteredRows.accept(project, visitor);
            jsonWriter.endArray();
            jsonWriter.key("filtered");
            jsonWriter.value(rwv.total);
            jsonWriter.key("total");
            jsonWriter.value(project.rows.size());
        } else {
            FilteredRecords filteredRecords = engine.getFilteredRecords();
            RecordVisitor visitor = rwv;
            if (sortingJson != null) {
                SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
                srv.initializeFromJSON(project, sortingJson);
                if (srv.hasCriteria()) {
                    visitor = srv;
                }
            }
            jsonWriter.key("mode");
            jsonWriter.value("record-based");
            jsonWriter.key("rows");
            jsonWriter.array();
            filteredRecords.accept(project, visitor);
            jsonWriter.endArray();
            jsonWriter.key("filtered");
            jsonWriter.value(rwv.total);
            jsonWriter.key("total");
            jsonWriter.value(project.recordModel.getRecordCount());
        }
        jsonWriter.key("start");
        jsonWriter.value(start);
        jsonWriter.key("limit");
        jsonWriter.value(limit);
        jsonWriter.key("pool");
        pool.write(jsonWriter, options);
        jsonWriter.endObject();
        if (callback != null) {
            writer.write(")");
        }
    } catch (Exception e) {
        respondException(response, e);
    }
}

Example 2 with FilteredRows

use of com.google.refine.browsing.FilteredRows in project OpenRefine by OpenRefine.

the class CustomizableTabularExporterUtilities method exportRows.

public static void exportRows(final Project project, final Engine engine, Properties params, final TabularSerializer serializer) {
    String optionsString = (params != null) ? params.getProperty("options") : null;
    JSONObject optionsTemp = null;
    if (optionsString != null) {
        try {
            optionsTemp = ParsingUtilities.evaluateJsonStringToObject(optionsString);
        } catch (JSONException e) {
        // Ignore and keep options null.
        }
    }
    final JSONObject options = optionsTemp;
    final boolean outputColumnHeaders = options == null ? true : JSONUtilities.getBoolean(options, "outputColumnHeaders", true);
    final boolean outputEmptyRows = options == null ? false : JSONUtilities.getBoolean(options, "outputBlankRows", true);
    final int limit = options == null ? -1 : JSONUtilities.getInt(options, "limit", -1);
    final List<String> columnNames;
    final Map<String, CellFormatter> columnNameToFormatter = new HashMap<String, CustomizableTabularExporterUtilities.CellFormatter>();
    JSONArray columnOptionArray = options == null ? null : JSONUtilities.getArray(options, "columns");
    if (columnOptionArray == null) {
        List<Column> columns = project.columnModel.columns;
        columnNames = new ArrayList<String>(columns.size());
        for (Column column : columns) {
            String name = column.getName();
            columnNames.add(name);
            columnNameToFormatter.put(name, new CellFormatter());
        }
    } else {
        int count = columnOptionArray.length();
        columnNames = new ArrayList<String>(count);
        for (int i = 0; i < count; i++) {
            JSONObject columnOptions = JSONUtilities.getObjectElement(columnOptionArray, i);
            if (columnOptions != null) {
                String name = JSONUtilities.getString(columnOptions, "name", null);
                if (name != null) {
                    columnNames.add(name);
                    columnNameToFormatter.put(name, new CellFormatter(columnOptions));
                }
            }
        }
    }
    RowVisitor visitor = new RowVisitor() {

        int rowCount = 0;

        @Override
        public void start(Project project) {
            serializer.startFile(options);
            if (outputColumnHeaders) {
                List<CellData> cells = new ArrayList<TabularSerializer.CellData>(columnNames.size());
                for (String name : columnNames) {
                    cells.add(new CellData(name, name, name, null));
                }
                serializer.addRow(cells, true);
            }
        }

        @Override
        public boolean visit(Project project, int rowIndex, Row row) {
            List<CellData> cells = new ArrayList<TabularSerializer.CellData>(columnNames.size());
            int nonNullCount = 0;
            for (String columnName : columnNames) {
                Column column = project.columnModel.getColumnByName(columnName);
                CellFormatter formatter = columnNameToFormatter.get(columnName);
                CellData cellData = formatter.format(project, column, row.getCell(column.getCellIndex()));
                cells.add(cellData);
                if (cellData != null) {
                    nonNullCount++;
                }
            }
            if (nonNullCount > 0 || outputEmptyRows) {
                serializer.addRow(cells, false);
                rowCount++;
            }
            return limit > 0 && rowCount >= limit;
        }

        @Override
        public void end(Project project) {
            serializer.endFile();
        }
    };
    FilteredRows filteredRows = engine.getAllFilteredRows();
    filteredRows.accept(project, visitor);
}

Example 3 with FilteredRows

use of com.google.refine.browsing.FilteredRows in project OpenRefine by OpenRefine.

the class TemplatingExporter method export.

@Override
public void export(Project project, Properties options, Engine engine, Writer writer) throws IOException {
    String limitString = options.getProperty("limit");
    int limit = limitString != null ? Integer.parseInt(limitString) : -1;
    JSONObject sortingJson = null;
    try {
        String json = options.getProperty("sorting");
        sortingJson = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json);
    } catch (JSONException e) {
    }
    String templateString = options.getProperty("template");
    String prefixString = options.getProperty("prefix");
    String suffixString = options.getProperty("suffix");
    String separatorString = options.getProperty("separator");
    Template template;
    try {
        template = Parser.parse(templateString);
    } catch (ParsingException e) {
        throw new IOException("Missing or bad template", e);
    }
    template.setPrefix(prefixString);
    template.setSuffix(suffixString);
    template.setSeparator(separatorString);
    if (!"true".equals(options.getProperty("preview"))) {
        StringWriter stringWriter = new StringWriter();
        JSONWriter jsonWriter = new JSONWriter(stringWriter);
        try {
            jsonWriter.object();
            jsonWriter.key("template");
            jsonWriter.value(templateString);
            jsonWriter.key("prefix");
            jsonWriter.value(prefixString);
            jsonWriter.key("suffix");
            jsonWriter.value(suffixString);
            jsonWriter.key("separator");
            jsonWriter.value(separatorString);
            jsonWriter.endObject();
        } catch (JSONException e) {
        // ignore
        }
        project.getMetadata().getPreferenceStore().put("exporters.templating.template", stringWriter.toString());
    }
    if (engine.getMode() == Mode.RowBased) {
        FilteredRows filteredRows = engine.getAllFilteredRows();
        RowVisitor visitor = template.getRowVisitor(writer, limit);
        if (sortingJson != null) {
            try {
                SortingRowVisitor srv = new SortingRowVisitor(visitor);
                srv.initializeFromJSON(project, sortingJson);
                if (srv.hasCriteria()) {
                    visitor = srv;
                }
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        filteredRows.accept(project, visitor);
    } else {
        FilteredRecords filteredRecords = engine.getFilteredRecords();
        RecordVisitor visitor = template.getRecordVisitor(writer, limit);
        if (sortingJson != null) {
            try {
                SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
                srv.initializeFromJSON(project, sortingJson);
                if (srv.hasCriteria()) {
                    visitor = srv;
                }
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        filteredRecords.accept(project, visitor);
    }
}

Example 4 with FilteredRows

use of com.google.refine.browsing.FilteredRows in project OpenRefine by OpenRefine.

the class ColumnAdditionOperation method createHistoryEntry.

@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
    Engine engine = createEngine(project);
    Column column = project.columnModel.getColumnByName(_baseColumnName);
    if (column == null) {
        throw new Exception("No column named " + _baseColumnName);
    }
    if (project.columnModel.getColumnByName(_newColumnName) != null) {
        throw new Exception("Another column already named " + _newColumnName);
    }
    List<CellAtRow> cellsAtRows = new ArrayList<CellAtRow>(project.rows.size());
    FilteredRows filteredRows = engine.getAllFilteredRows();
    filteredRows.accept(project, createRowVisitor(project, cellsAtRows));
    String description = createDescription(column, cellsAtRows);
    Change change = new ColumnAdditionChange(_newColumnName, _columnInsertIndex, cellsAtRows);
    return new HistoryEntry(historyEntryID, project, description, this, change);
}

Example 5 with FilteredRows

use of com.google.refine.browsing.FilteredRows in project OpenRefine by OpenRefine.

the class ColumnSplitOperation method createHistoryEntry.

@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
    Engine engine = createEngine(project);
    Column column = project.columnModel.getColumnByName(_columnName);
    if (column == null) {
        throw new Exception("No column named " + _columnName);
    }
    List<String> columnNames = new ArrayList<String>();
    List<Integer> rowIndices = new ArrayList<Integer>(project.rows.size());
    List<List<Serializable>> tuples = new ArrayList<List<Serializable>>(project.rows.size());
    FilteredRows filteredRows = engine.getAllFilteredRows();
    RowVisitor rowVisitor;
    if ("lengths".equals(_mode)) {
        rowVisitor = new ColumnSplitRowVisitor(column.getCellIndex(), columnNames, rowIndices, tuples) {

            @Override
            protected java.util.List<Serializable> split(String s) {
                List<Serializable> results = new ArrayList<Serializable>(_fieldLengths.length + 1);
                int lastIndex = 0;
                for (int length : _fieldLengths) {
                    int from = lastIndex;
                    int to = Math.min(from + length, s.length());
                    results.add(stringToValue(s.substring(from, to)));
                    lastIndex = to;
                }
                return results;
            }

            ;
        };
    } else if (_regex) {
        Pattern pattern = Pattern.compile(_separator);
        rowVisitor = new ColumnSplitRowVisitor(column.getCellIndex(), columnNames, rowIndices, tuples) {

            Pattern _pattern;

            @Override
            protected java.util.List<Serializable> split(String s) {
                return stringArrayToValueList(_pattern.split(s, _maxColumns));
            }

            ;

            public RowVisitor init(Pattern pattern) {
                _pattern = pattern;
                return this;
            }
        }.init(pattern);
    } else {
        rowVisitor = new ColumnSplitRowVisitor(column.getCellIndex(), columnNames, rowIndices, tuples) {

            @Override
            protected java.util.List<Serializable> split(String s) {
                return stringArrayToValueList(StringUtils.splitByWholeSeparatorPreserveAllTokens(s, _separator, _maxColumns));
            }

            ;
        };
    }
    filteredRows.accept(project, rowVisitor);
    String description = "Split " + rowIndices.size() + " cell(s) in column " + _columnName + " into several columns" + ("separator".equals(_mode) ? " by separator" : " by field lengths");
    Change change = new ColumnSplitChange(_columnName, columnNames, rowIndices, tuples, _removeOriginalColumn);
    return new HistoryEntry(historyEntryID, project, description, this, change);
}