Examples with Progress com.google.refine.importing.ImportingUtilities.Progress used on opensource projects

Example 1 with Progress

use of com.google.refine.importing.ImportingUtilities.Progress in project polymap4-core by Polymap4.

the class FilebasedImportingController method loadDataAndPrepareJob.

// based on
// com.google.refine.importing.ImportingUtilities.loadDataAndPrepareJob(HttpServletRequest,
// HttpServletResponse, Properties, ImportingJob, JSONObject)
private void loadDataAndPrepareJob(RefineRequest request, HttpServletResponse response, Properties parameters, final ImportingJob job, JSONObject config) throws IOException, ServletException {
    JSONObject retrievalRecord = new JSONObject();
    JSONUtilities.safePut(config, "retrievalRecord", retrievalRecord);
    JSONUtilities.safePut(config, "state", "loading-raw-data");
    final JSONObject progress = new JSONObject();
    JSONUtilities.safePut(config, "progress", progress);
    try {
        retrieveContentFromRefineRequest(request, parameters, job.getRawDataDir(), retrievalRecord, new Progress() {

            @Override
            public void setProgress(String message, int percent) {
                if (message != null) {
                    JSONUtilities.safePut(progress, "message", message);
                }
                JSONUtilities.safePut(progress, "percent", percent);
            }

            @Override
            public boolean isCanceled() {
                return job.canceled;
            }
        });
    } catch (Exception e) {
        log.error("Error uploading data", e);
        JSONUtilities.safePut(config, "state", "error");
        JSONUtilities.safePut(config, "error", "Error uploading data");
        JSONUtilities.safePut(config, "errorDetails", e.getLocalizedMessage());
        return;
    }
    JSONArray fileSelectionIndexes = new JSONArray();
    JSONUtilities.safePut(config, "fileSelection", fileSelectionIndexes);
    EncodingGuesser.guess(job);
    String bestFormat = ImportingUtilities.autoSelectFiles(job, retrievalRecord, fileSelectionIndexes);
    bestFormat = ImportingUtilities.guessBetterFormat(job, bestFormat);
    JSONArray rankedFormats = new JSONArray();
    ImportingUtilities.rankFormats(job, bestFormat, rankedFormats);
    JSONUtilities.safePut(config, "rankedFormats", rankedFormats);
    JSONUtilities.safePut(config, "state", "ready");
    JSONUtilities.safePut(config, "hasData", true);
    config.remove("progress");
}

Example 2 with Progress

use of com.google.refine.importing.ImportingUtilities.Progress in project OpenRefine by OpenRefine.

the class ImportingUtilitiesTests method importUnsupportedZipFile.

@Test
public void importUnsupportedZipFile() throws IOException {
    String filename = "unsupportedPPMD.zip";
    String filepath = ClassLoader.getSystemResource(filename).getPath();
    // Make a copy in our data directory where it's expected
    File tmp = File.createTempFile("openrefine-test-unsupportedPPMD", ".zip", job.getRawDataDir());
    tmp.deleteOnExit();
    FileUtils.copyFile(new File(filepath), tmp);
    Progress dummyProgress = new Progress() {

        @Override
        public void setProgress(String message, int percent) {
        }

        @Override
        public boolean isCanceled() {
            return false;
        }
    };
    ArrayNode fileRecords = ParsingUtilities.mapper.createArrayNode();
    ObjectNode fileRecord = ParsingUtilities.mapper.createObjectNode();
    JSONUtilities.safePut(fileRecord, "origin", "upload");
    JSONUtilities.safePut(fileRecord, "declaredEncoding", "UTF-8");
    JSONUtilities.safePut(fileRecord, "declaredMimeType", "application/x-zip-compressed");
    JSONUtilities.safePut(fileRecord, "fileName", filename);
    JSONUtilities.safePut(fileRecord, "location", tmp.getName());
    HttpServletRequest request = mock(HttpServletRequest.class);
    HttpServletResponse response = mock(HttpServletResponse.class);
    assertThrows(IOException.class, () -> ImportingUtilities.postProcessRetrievedFile(job.getRawDataDir(), tmp, fileRecord, fileRecords, dummyProgress));
    assertThrows(FileUploadBase.InvalidContentTypeException.class, () -> ImportingUtilities.retrieveContentFromPostRequest(request, new Properties(), job.getRawDataDir(), fileRecord, dummyProgress));
    assertThrows(IOException.class, () -> ImportingUtilities.loadDataAndPrepareJob(request, response, new Properties(), job, fileRecord));
}

Example 3 with Progress

use of com.google.refine.importing.ImportingUtilities.Progress in project OpenRefine by OpenRefine.

the class ImportingUtilitiesTests method importArchive.

/**
 * This tests both exploding a zip archive into it's constituent files as well as importing them all (both) and
 * making sure that the recording of archive names and file names works correctly.
 *
 * It's kind of a lot to have in one test, but it's a sequence of steps that need to be done in order.
 *
 * @throws IOException
 */
@SuppressWarnings("unchecked")
@Test
public void importArchive() throws IOException {
    String filename = "movies.zip";
    String filepath = ClassLoader.getSystemResource(filename).getPath();
    // Make a copy in our data directory where it's expected
    File tmp = File.createTempFile("openrefine-test-movies", ".zip", job.getRawDataDir());
    tmp.deleteOnExit();
    FileUtils.copyFile(new File(filepath), tmp);
    Progress dummyProgress = new Progress() {

        @Override
        public void setProgress(String message, int percent) {
        }

        @Override
        public boolean isCanceled() {
            return false;
        }
    };
    ArrayNode fileRecords = ParsingUtilities.mapper.createArrayNode();
    ObjectNode fileRecord = ParsingUtilities.mapper.createObjectNode();
    JSONUtilities.safePut(fileRecord, "origin", "upload");
    JSONUtilities.safePut(fileRecord, "declaredEncoding", "UTF-8");
    JSONUtilities.safePut(fileRecord, "declaredMimeType", "application/x-zip-compressed");
    JSONUtilities.safePut(fileRecord, "fileName", filename);
    JSONUtilities.safePut(fileRecord, "location", tmp.getName());
    assertTrue(ImportingUtilities.postProcessRetrievedFile(job.getRawDataDir(), tmp, fileRecord, fileRecords, dummyProgress));
    assertEquals(fileRecords.size(), 2);
    assertEquals(fileRecords.get(0).get("fileName").asText(), "movies-condensed.tsv");
    assertEquals(fileRecords.get(0).get("archiveFileName").asText(), "movies.zip");
    assertEquals(fileRecords.get(1).get("fileName").asText(), "movies.tsv");
    ObjectNode options = ParsingUtilities.mapper.createObjectNode();
    JSONUtilities.safePut(options, "includeArchiveFileName", true);
    JSONUtilities.safePut(options, "includeFileSources", true);
    ImportingParserBase parser = new SeparatorBasedImporter();
    List<Exception> exceptions = new ArrayList<Exception>();
    parser.parse(project, metadata, job, IteratorUtils.toList(fileRecords.iterator()), "tsv", -1, options, exceptions);
    assertEquals(exceptions.size(), 0);
    project.update();
    assertEquals(project.columnModel.columns.get(0).getName(), "Archive");
    assertEquals(project.rows.get(0).getCell(0).getValue(), "movies.zip");
    assertEquals(project.columnModel.columns.get(1).getName(), "File");
    assertEquals(project.rows.get(0).getCell(1).getValue(), "movies-condensed.tsv");
    assertEquals(project.columnModel.columns.get(2).getName(), "name");
    assertEquals(project.rows.get(0).getCell(2).getValue(), "Wayne's World");
    // Make sure we imported both files contained in the zip file
    assertEquals(project.rows.size(), 252);
    ArrayNode importOptionsArray = metadata.getImportOptionMetadata();
    assertEquals(importOptionsArray.size(), 2);
    ObjectNode importOptions = (ObjectNode) importOptionsArray.get(0);
    assertEquals(importOptions.get("archiveFileName").asText(), "movies.zip");
    assertEquals(importOptions.get("fileSource").asText(), "movies-condensed.tsv");
    assertTrue(importOptions.get("includeFileSources").asBoolean());
    assertTrue(importOptions.get("includeArchiveFileName").asBoolean());
    importOptions = (ObjectNode) importOptionsArray.get(1);
    assertEquals(importOptions.get("fileSource").asText(), "movies.tsv");
    assertEquals(importOptions.get("archiveFileName").asText(), "movies.zip");
}

Example 4 with Progress

use of com.google.refine.importing.ImportingUtilities.Progress in project OpenRefine by OpenRefine.

the class ImportingUtilitiesTests method urlImporting.

@Test
public void urlImporting() throws IOException {
    String RESPONSE_BODY = "{code:401,message:Unauthorised}";
    MockWebServer server = new MockWebServer();
    MockResponse mockResponse = new MockResponse();
    mockResponse.setBody(RESPONSE_BODY);
    mockResponse.setResponseCode(401);
    server.start();
    server.enqueue(mockResponse);
    HttpUrl url = server.url("/random");
    String MESSAGE = String.format("HTTP error %d : %s for URL %s", 401, "Client Error", url);
    MultipartEntityBuilder builder = MultipartEntityBuilder.create();
    StringBody stringBody = new StringBody(url.toString(), ContentType.MULTIPART_FORM_DATA);
    builder = builder.addPart("download", stringBody);
    HttpEntity entity = builder.build();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    entity.writeTo(os);
    ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray());
    HttpServletRequest req = mock(HttpServletRequest.class);
    when(req.getContentType()).thenReturn(entity.getContentType().getValue());
    when(req.getParameter("download")).thenReturn(url.toString());
    when(req.getMethod()).thenReturn("POST");
    when(req.getContentLength()).thenReturn((int) entity.getContentLength());
    when(req.getInputStream()).thenReturn(new MockServletInputStream(is));
    ImportingJob job = ImportingManager.createJob();
    Properties parameters = ParsingUtilities.parseUrlParameters(req);
    ObjectNode retrievalRecord = ParsingUtilities.mapper.createObjectNode();
    ObjectNode progress = ParsingUtilities.mapper.createObjectNode();
    try {
        ImportingUtilities.retrieveContentFromPostRequest(req, parameters, job.getRawDataDir(), retrievalRecord, new ImportingUtilities.Progress() {

            @Override
            public void setProgress(String message, int percent) {
                if (message != null) {
                    JSONUtilities.safePut(progress, "message", message);
                }
                JSONUtilities.safePut(progress, "percent", percent);
            }

            @Override
            public boolean isCanceled() {
                return job.canceled;
            }
        });
        fail("No Exception was thrown");
    } catch (Exception exception) {
        assertEquals(exception.getMessage(), MESSAGE);
    } finally {
        server.close();
    }
}