Examples with ImportingParserBase com.google.refine.importers.ImportingParserBase used on opensource projects

Example 1 with ImportingParserBase

use of com.google.refine.importers.ImportingParserBase in project OpenRefine by OpenRefine.

the class ImportingUtilitiesTests method importArchive.

/**
 * This tests both exploding a zip archive into it's constituent files as well as importing them all (both) and
 * making sure that the recording of archive names and file names works correctly.
 *
 * It's kind of a lot to have in one test, but it's a sequence of steps that need to be done in order.
 *
 * @throws IOException
 */
@SuppressWarnings("unchecked")
@Test
public void importArchive() throws IOException {
    String filename = "movies.zip";
    String filepath = ClassLoader.getSystemResource(filename).getPath();
    // Make a copy in our data directory where it's expected
    File tmp = File.createTempFile("openrefine-test-movies", ".zip", job.getRawDataDir());
    tmp.deleteOnExit();
    FileUtils.copyFile(new File(filepath), tmp);
    Progress dummyProgress = new Progress() {

        @Override
        public void setProgress(String message, int percent) {
        }

        @Override
        public boolean isCanceled() {
            return false;
        }
    };
    ArrayNode fileRecords = ParsingUtilities.mapper.createArrayNode();
    ObjectNode fileRecord = ParsingUtilities.mapper.createObjectNode();
    JSONUtilities.safePut(fileRecord, "origin", "upload");
    JSONUtilities.safePut(fileRecord, "declaredEncoding", "UTF-8");
    JSONUtilities.safePut(fileRecord, "declaredMimeType", "application/x-zip-compressed");
    JSONUtilities.safePut(fileRecord, "fileName", filename);
    JSONUtilities.safePut(fileRecord, "location", tmp.getName());
    assertTrue(ImportingUtilities.postProcessRetrievedFile(job.getRawDataDir(), tmp, fileRecord, fileRecords, dummyProgress));
    assertEquals(fileRecords.size(), 2);
    assertEquals(fileRecords.get(0).get("fileName").asText(), "movies-condensed.tsv");
    assertEquals(fileRecords.get(0).get("archiveFileName").asText(), "movies.zip");
    assertEquals(fileRecords.get(1).get("fileName").asText(), "movies.tsv");
    ObjectNode options = ParsingUtilities.mapper.createObjectNode();
    JSONUtilities.safePut(options, "includeArchiveFileName", true);
    JSONUtilities.safePut(options, "includeFileSources", true);
    ImportingParserBase parser = new SeparatorBasedImporter();
    List<Exception> exceptions = new ArrayList<Exception>();
    parser.parse(project, metadata, job, IteratorUtils.toList(fileRecords.iterator()), "tsv", -1, options, exceptions);
    assertEquals(exceptions.size(), 0);
    project.update();
    assertEquals(project.columnModel.columns.get(0).getName(), "Archive");
    assertEquals(project.rows.get(0).getCell(0).getValue(), "movies.zip");
    assertEquals(project.columnModel.columns.get(1).getName(), "File");
    assertEquals(project.rows.get(0).getCell(1).getValue(), "movies-condensed.tsv");
    assertEquals(project.columnModel.columns.get(2).getName(), "name");
    assertEquals(project.rows.get(0).getCell(2).getValue(), "Wayne's World");
    // Make sure we imported both files contained in the zip file
    assertEquals(project.rows.size(), 252);
    ArrayNode importOptionsArray = metadata.getImportOptionMetadata();
    assertEquals(importOptionsArray.size(), 2);
    ObjectNode importOptions = (ObjectNode) importOptionsArray.get(0);
    assertEquals(importOptions.get("archiveFileName").asText(), "movies.zip");
    assertEquals(importOptions.get("fileSource").asText(), "movies-condensed.tsv");
    assertTrue(importOptions.get("includeFileSources").asBoolean());
    assertTrue(importOptions.get("includeArchiveFileName").asBoolean());
    importOptions = (ObjectNode) importOptionsArray.get(1);
    assertEquals(importOptions.get("fileSource").asText(), "movies.tsv");
    assertEquals(importOptions.get("archiveFileName").asText(), "movies.zip");
}