Examples with DataExtension com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension used on opensource projects

Example 1 with DataExtension

use of com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension in project OpenRefine by OpenRefine.

the class PreviewExtendDataCommand method doPost.

@Override
public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (!hasValidCSRFToken(request)) {
        respondCSRFError(response);
        return;
    }
    try {
        Project project = getProject(request);
        String columnName = request.getParameter("columnName");
        String rowIndicesString = request.getParameter("rowIndices");
        if (rowIndicesString == null) {
            respond(response, "{ \"code\" : \"error\", \"message\" : \"No row indices specified\" }");
            return;
        }
        String jsonString = request.getParameter("extension");
        DataExtensionConfig config = DataExtensionConfig.reconstruct(jsonString);
        List<Integer> rowIndices = ParsingUtilities.mapper.readValue(rowIndicesString, new TypeReference<List<Integer>>() {
        });
        int length = rowIndices.size();
        Column column = project.columnModel.getColumnByName(columnName);
        int cellIndex = column.getCellIndex();
        // get the endpoint to extract data from
        String endpoint = null;
        ReconConfig cfg = column.getReconConfig();
        if (cfg != null && cfg instanceof StandardReconConfig) {
            StandardReconConfig scfg = (StandardReconConfig) cfg;
            endpoint = scfg.service;
        } else {
            respond(response, "{ \"code\" : \"error\", \"message\" : \"This column has not been reconciled with a standard service.\" }");
            return;
        }
        List<String> topicNames = new ArrayList<String>();
        List<String> topicIds = new ArrayList<String>();
        Set<String> ids = new HashSet<String>();
        for (int i = 0; i < length; i++) {
            int rowIndex = rowIndices.get(i);
            if (rowIndex >= 0 && rowIndex < project.rows.size()) {
                Row row = project.rows.get(rowIndex);
                Cell cell = row.getCell(cellIndex);
                if (cell != null && cell.recon != null && cell.recon.match != null) {
                    topicNames.add(cell.recon.match.name);
                    topicIds.add(cell.recon.match.id);
                    ids.add(cell.recon.match.id);
                } else {
                    topicNames.add(null);
                    topicIds.add(null);
                    ids.add(null);
                }
            }
        }
        Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
        ReconciledDataExtensionJob job = new ReconciledDataExtensionJob(config, endpoint);
        Map<String, DataExtension> map = job.extend(ids, reconCandidateMap);
        List<List<Object>> rows = new ArrayList<>();
        for (int r = 0; r < topicNames.size(); r++) {
            String id = topicIds.get(r);
            String topicName = topicNames.get(r);
            if (id != null && map.containsKey(id)) {
                DataExtension ext = map.get(id);
                boolean first = true;
                if (ext.data.length > 0) {
                    for (Object[] row : ext.data) {
                        List<Object> jsonRow = new ArrayList<>();
                        if (first) {
                            jsonRow.add(topicName);
                            first = false;
                        } else {
                            jsonRow.add(null);
                        }
                        for (Object cell : row) {
                            jsonRow.add(cell);
                        }
                        rows.add(jsonRow);
                    }
                    continue;
                }
            }
            List<Object> supplement = new ArrayList<>();
            if (id != null) {
                supplement.add(new ReconCandidate(id, topicName, new String[0], 100));
            } else {
                supplement.add("<not reconciled>");
            }
            rows.add(supplement);
        }
        respondJSON(response, new PreviewResponse(job.columns, rows));
    } catch (Exception e) {
        respondException(response, e);
    }
}

Example 2 with DataExtension

use of com.google.refine.model.recon.ReconciledDataExtensionJob.DataExtension in project OpenRefine by OpenRefine.

the class DataExtensionChange method apply.

@Override
public void apply(Project project) {
    synchronized (project) {
        if (_firstNewCellIndex < 0) {
            _firstNewCellIndex = project.columnModel.allocateNewCellIndex();
            for (int i = 1; i < _columnNames.size(); i++) {
                project.columnModel.allocateNewCellIndex();
            }
            _oldRows = new ArrayList<Row>(project.rows);
            _newRows = new ArrayList<Row>(project.rows.size());
            int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
            int keyCellIndex = project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
            int index = 0;
            int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
            DataExtension dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
            index++;
            Map<String, Recon> reconMap = new HashMap<String, Recon>();
            for (int r = 0; r < _oldRows.size(); r++) {
                Row oldRow = _oldRows.get(r);
                if (r < rowIndex) {
                    _newRows.add(oldRow.dup());
                    continue;
                }
                if (dataExtension == null || dataExtension.data.length == 0) {
                    _newRows.add(oldRow);
                } else {
                    Row firstNewRow = oldRow.dup();
                    extendRow(firstNewRow, dataExtension, 0, reconMap);
                    _newRows.add(firstNewRow);
                    int r2 = r + 1;
                    for (int subR = 1; subR < dataExtension.data.length; subR++) {
                        if (r2 < project.rows.size()) {
                            Row oldRow2 = project.rows.get(r2);
                            if (oldRow2.isCellBlank(cellIndex) && oldRow2.isCellBlank(keyCellIndex)) {
                                Row newRow = oldRow2.dup();
                                extendRow(newRow, dataExtension, subR, reconMap);
                                _newRows.add(newRow);
                                r2++;
                                continue;
                            }
                        }
                        Row newRow = new Row(cellIndex + _columnNames.size());
                        extendRow(newRow, dataExtension, subR, reconMap);
                        _newRows.add(newRow);
                    }
                    // r will be incremented by the for loop anyway
                    r = r2 - 1;
                }
                rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
                dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
                index++;
            }
        }
        project.rows.clear();
        project.rows.addAll(_newRows);
        for (int i = 0; i < _columnNames.size(); i++) {
            String name = _columnNames.get(i);
            int cellIndex = _firstNewCellIndex + i;
            Column column = new Column(cellIndex, name);
            ReconType columnType = _columnTypes.get(i);
            column.setReconConfig(new DataExtensionReconConfig(_service, _identifierSpace, _schemaSpace, columnType));
            ReconStats reconStats = ReconStats.create(project, cellIndex);
            if (reconStats.matchedTopics > 0) {
                column.setReconStats(reconStats);
            }
            try {
                project.columnModel.addColumn(_columnInsertIndex + i, column, true);
                // the column might have been renamed to avoid collision
                _columnNames.set(i, column.getName());
            } catch (ModelException e) {
            // won't get here since we set the avoid collision flag
            }
        }
        project.update();
    }
}