Examples with ImportingJob com.google.refine.importing.ImportingJob used on opensource projects

Example 1 with ImportingJob

use of com.google.refine.importing.ImportingJob in project OpenRefine by OpenRefine.

the class GetRowsCommand method internalRespond.

protected void internalRespond(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    try {
        Project project = null;
        // This command also supports retrieving rows for an importing job.
        String importingJobID = request.getParameter("importingJobID");
        if (importingJobID != null) {
            long jobID = Long.parseLong(importingJobID);
            ImportingJob job = ImportingManager.getJob(jobID);
            if (job != null) {
                project = job.project;
            }
        }
        if (project == null) {
            project = getProject(request);
        }
        Engine engine = getEngine(request, project);
        String callback = request.getParameter("callback");
        int start = Math.min(project.rows.size(), Math.max(0, getIntegerParameter(request, "start", 0)));
        int limit = Math.min(project.rows.size() - start, Math.max(0, getIntegerParameter(request, "limit", 20)));
        Pool pool = new Pool();
        Properties options = new Properties();
        options.put("project", project);
        options.put("reconCandidateOmitTypes", true);
        options.put("pool", pool);
        response.setCharacterEncoding("UTF-8");
        response.setHeader("Content-Type", callback == null ? "application/json" : "text/javascript");
        PrintWriter writer = response.getWriter();
        if (callback != null) {
            writer.write(callback);
            writer.write("(");
        }
        JSONWriter jsonWriter = new JSONWriter(writer);
        jsonWriter.object();
        RowWritingVisitor rwv = new RowWritingVisitor(start, limit, jsonWriter, options);
        JSONObject sortingJson = null;
        try {
            String json = request.getParameter("sorting");
            sortingJson = (json == null) ? null : ParsingUtilities.evaluateJsonStringToObject(json);
        } catch (JSONException e) {
        }
        if (engine.getMode() == Mode.RowBased) {
            FilteredRows filteredRows = engine.getAllFilteredRows();
            RowVisitor visitor = rwv;
            if (sortingJson != null) {
                SortingRowVisitor srv = new SortingRowVisitor(visitor);
                srv.initializeFromJSON(project, sortingJson);
                if (srv.hasCriteria()) {
                    visitor = srv;
                }
            }
            jsonWriter.key("mode");
            jsonWriter.value("row-based");
            jsonWriter.key("rows");
            jsonWriter.array();
            filteredRows.accept(project, visitor);
            jsonWriter.endArray();
            jsonWriter.key("filtered");
            jsonWriter.value(rwv.total);
            jsonWriter.key("total");
            jsonWriter.value(project.rows.size());
        } else {
            FilteredRecords filteredRecords = engine.getFilteredRecords();
            RecordVisitor visitor = rwv;
            if (sortingJson != null) {
                SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
                srv.initializeFromJSON(project, sortingJson);
                if (srv.hasCriteria()) {
                    visitor = srv;
                }
            }
            jsonWriter.key("mode");
            jsonWriter.value("record-based");
            jsonWriter.key("rows");
            jsonWriter.array();
            filteredRecords.accept(project, visitor);
            jsonWriter.endArray();
            jsonWriter.key("filtered");
            jsonWriter.value(rwv.total);
            jsonWriter.key("total");
            jsonWriter.value(project.recordModel.getRecordCount());
        }
        jsonWriter.key("start");
        jsonWriter.value(start);
        jsonWriter.key("limit");
        jsonWriter.value(limit);
        jsonWriter.key("pool");
        pool.write(jsonWriter, options);
        jsonWriter.endObject();
        if (callback != null) {
            writer.write(")");
        }
    } catch (Exception e) {
        respondException(response, e);
    }
}

Example 2 with ImportingJob

use of com.google.refine.importing.ImportingJob in project OpenRefine by OpenRefine.

the class CreateProjectCommand method doPost.

@Override
public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    ProjectManager.singleton.setBusy(true);
    try {
        Properties parameters = ParsingUtilities.parseUrlParameters(request);
        ImportingJob job = ImportingManager.createJob();
        JSONObject config = job.getOrCreateDefaultConfig();
        ImportingUtilities.loadDataAndPrepareJob(request, response, parameters, job, config);
        String format = parameters.getProperty("format");
        // to check if we have a parser for it. If not, null it out.
        if (format != null && !format.isEmpty()) {
            Format formatRecord = ImportingManager.formatToRecord.get(format);
            if (formatRecord == null || formatRecord.parser == null) {
                format = null;
            }
        }
        // If we don't have a format specified, try to guess it.
        if (format == null || format.isEmpty()) {
            // Use legacy parameters to guess the format.
            if ("false".equals(parameters.getProperty("split-into-columns"))) {
                format = "text/line-based";
            } else if (",".equals(parameters.getProperty("separator")) || "\\t".equals(parameters.getProperty("separator"))) {
                format = "text/line-based/*sv";
            } else {
                JSONArray rankedFormats = JSONUtilities.getArray(config, "rankedFormats");
                if (rankedFormats != null && rankedFormats.length() > 0) {
                    format = rankedFormats.getString(0);
                }
            }
            if (format == null || format.isEmpty()) {
                // If we have failed in guessing, default to something simple.
                format = "text/line-based";
            }
        }
        JSONObject optionObj = null;
        String optionsString = request.getParameter("options");
        if (optionsString != null && !optionsString.isEmpty()) {
            optionObj = ParsingUtilities.evaluateJsonStringToObject(optionsString);
        } else {
            Format formatRecord = ImportingManager.formatToRecord.get(format);
            optionObj = formatRecord.parser.createParserUIInitializationData(job, job.getSelectedFileRecords(), format);
        }
        adjustLegacyOptions(format, parameters, optionObj);
        String projectName = parameters.getProperty("project-name");
        if (projectName != null && !projectName.isEmpty()) {
            JSONUtilities.safePut(optionObj, "projectName", projectName);
        }
        List<Exception> exceptions = new LinkedList<Exception>();
        long projectId = ImportingUtilities.createProject(job, format, optionObj, exceptions, true);
        HttpUtilities.redirect(response, "/project?project=" + projectId);
    } catch (Exception e) {
        respondWithErrorPage(request, response, "Failed to import file", e);
    } finally {
        ProjectManager.singleton.setBusy(false);
    }
}

Example 3 with ImportingJob

use of com.google.refine.importing.ImportingJob in project OpenRefine by OpenRefine.

the class GetModelsCommand method internalRespond.

protected void internalRespond(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    Project project = null;
    // This command also supports retrieving rows for an importing job.
    String importingJobID = request.getParameter("importingJobID");
    if (importingJobID != null) {
        long jobID = Long.parseLong(importingJobID);
        ImportingJob job = ImportingManager.getJob(jobID);
        if (job != null) {
            project = job.project;
        }
    }
    if (project == null) {
        project = getProject(request);
    }
    try {
        response.setCharacterEncoding("UTF-8");
        response.setHeader("Content-Type", "application/json");
        response.setHeader("Cache-Control", "no-cache");
        Properties options = new Properties();
        JSONWriter writer = new JSONWriter(response.getWriter());
        writer.object();
        writer.key("columnModel");
        project.columnModel.write(writer, options);
        writer.key("recordModel");
        project.recordModel.write(writer, options);
        writer.key("overlayModels");
        writer.object();
        for (String modelName : project.overlayModels.keySet()) {
            OverlayModel overlayModel = project.overlayModels.get(modelName);
            if (overlayModel != null) {
                writer.key(modelName);
                project.overlayModels.get(modelName).write(writer, options);
            }
        }
        writer.endObject();
        writer.key("scripting");
        writer.object();
        for (String languagePrefix : MetaParser.getLanguagePrefixes()) {
            LanguageInfo info = MetaParser.getLanguageInfo(languagePrefix);
            writer.key(languagePrefix);
            writer.object();
            writer.key("name");
            writer.value(info.name);
            writer.key("defaultExpression");
            writer.value(info.defaultExpression);
            writer.endObject();
        }
        writer.endObject();
        writer.endObject();
    } catch (JSONException e) {
        HttpUtilities.respondException(response, e);
    }
}

Example 4 with ImportingJob

use of com.google.refine.importing.ImportingJob in project OpenRefine by OpenRefine.

the class GDataImportingController method doCreateProject.

private void doCreateProject(HttpServletRequest request, HttpServletResponse response, Properties parameters) throws ServletException, IOException {
    final String token = TokenCookie.getToken(request);
    long jobID = Long.parseLong(parameters.getProperty("jobID"));
    final ImportingJob job = ImportingManager.getJob(jobID);
    if (job == null) {
        HttpUtilities.respond(response, "error", "No such import job");
        return;
    }
    job.updating = true;
    try {
        final JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(request.getParameter("options"));
        final List<Exception> exceptions = new LinkedList<Exception>();
        job.setState("creating-project");
        final Project project = new Project();
        new Thread() {

            @Override
            public void run() {
                ProjectMetadata pm = new ProjectMetadata();
                pm.setName(JSONUtilities.getString(optionObj, "projectName", "Untitled"));
                pm.setEncoding(JSONUtilities.getString(optionObj, "encoding", "UTF-8"));
                GDataImporter.parse(token, project, pm, job, -1, optionObj, exceptions);
                if (!job.canceled) {
                    if (exceptions.size() > 0) {
                        job.setError(exceptions);
                    } else {
                        // update all internal models, indexes, caches, etc.
                        project.update();
                        ProjectManager.singleton.registerProject(project, pm);
                        job.setState("created-project");
                        job.setProjectID(project.id);
                    }
                    job.touch();
                    job.updating = false;
                }
            }
        }.start();
        HttpUtilities.respond(response, "ok", "done");
    } catch (JSONException e) {
        throw new ServletException(e);
    }
}

Example 5 with ImportingJob

use of com.google.refine.importing.ImportingJob in project OpenRefine by OpenRefine.

the class GDataImportingController method doParsePreview.

private void doParsePreview(HttpServletRequest request, HttpServletResponse response, Properties parameters) throws ServletException, IOException {
    String token = TokenCookie.getToken(request);
    long jobID = Long.parseLong(parameters.getProperty("jobID"));
    ImportingJob job = ImportingManager.getJob(jobID);
    if (job == null) {
        HttpUtilities.respond(response, "error", "No such import job");
        return;
    }
    job.updating = true;
    try {
        JSONObject optionObj = ParsingUtilities.evaluateJsonStringToObject(request.getParameter("options"));
        List<Exception> exceptions = new LinkedList<Exception>();
        job.prepareNewProject();
        GDataImporter.parse(token, job.project, job.metadata, job, 100, optionObj, exceptions);
        Writer w = response.getWriter();
        JSONWriter writer = new JSONWriter(w);
        try {
            writer.object();
            if (exceptions.size() == 0) {
                // update all internal models, indexes, caches, etc.
                job.project.update();
                writer.key("status");
                writer.value("ok");
            } else {
                writer.key("status");
                writer.value("error");
                writer.key("errors");
                writer.array();
                DefaultImportingController.writeErrors(writer, exceptions);
                writer.endArray();
            }
            writer.endObject();
        } catch (JSONException e) {
            throw new ServletException(e);
        } finally {
            w.flush();
            w.close();
        }
    } catch (JSONException e) {
        throw new ServletException(e);
    } finally {
        job.touch();
        job.updating = false;
    }
}