Examples with I5ZReader net.idea.i5.io.I5ZReader used on opensource projects

Example 1 with I5ZReader

use of net.idea.i5.io.I5ZReader in project ambit-mirror by ideaconsult.

the class I5SubstanceWriterTest method testWriteMultipleFiles_i5d.

@Test
public void testWriteMultipleFiles_i5d() throws Exception {
    setUpDatabaseFromResource("ambit2/db/processors/test/empty-datasets.xml");
    IDatabaseConnection c = getConnection();
    ITable chemicals = c.createQueryTable("EXPECTED", "SELECT * FROM chemicals");
    Assert.assertEquals(0, chemicals.getRowCount());
    ITable strucs = c.createQueryTable("EXPECTED", "SELECT * FROM structure");
    Assert.assertEquals(0, strucs.getRowCount());
    ITable srcdataset = c.createQueryTable("EXPECTED", "SELECT * FROM src_dataset");
    Assert.assertEquals(0, srcdataset.getRowCount());
    ITable struc_src = c.createQueryTable("EXPECTED", "SELECT * FROM struc_dataset");
    Assert.assertEquals(0, struc_src.getRowCount());
    ITable property = c.createQueryTable("EXPECTED", "SELECT * FROM properties");
    Assert.assertEquals(0, property.getRowCount());
    ITable property_values = c.createQueryTable("EXPECTED", "SELECT * FROM property_values");
    Assert.assertEquals(0, property_values.getRowCount());
    /**
     * Now reading only substances and reference substances Document types:
     * EndpointStudyRecord: 877 AttachmentDocument: 5 LegalEntity: 1
     * ReferenceSubstance: 6 Substance: 1 EndpointRecord: 14
     */
    InputStream in = this.getClass().getClassLoader().getResourceAsStream("ambit2/db/substance/testNM.i5z");
    // InputStream in =
    // I5AmbitProcessor.class.getClassLoader().getResourceAsStream("net/idea/i5/_5/substance/i5z/IUC4-efdb21bb-e79f-3286-a988-b6f6944d3734.i5z");
    // InputStream in =
    // I5AmbitProcessor.class.getClassLoader().getResourceAsStream("net/idea/i5/_5/substance/i5z/IUC4-e2b69497-1c50-3d0b-a2b2-41d0a4d74c54.i5z");
    // InputStream in =
    // I5AmbitProcessor.class.getClassLoader().getResourceAsStream("net/idea/i5/_5/substance/i5z/IUC4-f5dd46ce-6fc9-316f-a468-c4f9acfcfc3c.i5z");
    Assert.assertNotNull(in);
    File i5z = File.createTempFile("test_", ".i5z");
    try {
        DownloadTool.download(in, i5z);
    } finally {
        in.close();
    }
    Assert.assertTrue(i5z.exists());
    I5ZReader reader = null;
    int records = 0;
    try {
        reader = new I5ZReader(i5z);
        QASettings qa = new QASettings(false);
        qa.setAll();
        reader.setQASettings(qa);
        PropertyKey key = new ReferenceSubstanceUUID();
        records = write(null, reader, c.getConnection(), key, true, false, false);
    } finally {
        try {
            reader.close();
        } catch (Exception x) {
        }
        try {
            c.close();
        } catch (Exception x) {
        }
        try {
            i5z.delete();
        } catch (Exception x) {
        }
    }
    Assert.assertEquals(8, records);
    c = getConnection();
    ITable substance = c.createQueryTable("EXPECTED", "SELECT * FROM substance");
    Assert.assertEquals(1, substance.getRowCount());
    Assert.assertNotNull(substance.getValue(0, "uuid"));
    chemicals = c.createQueryTable("EXPECTED", "SELECT * FROM chemicals");
    Assert.assertEquals(1, chemicals.getRowCount());
    // there are two empty file without $$$$ sign, which are skipped
    strucs = c.createQueryTable("EXPECTED", "SELECT * FROM structure");
    Assert.assertEquals(1, strucs.getRowCount());
    srcdataset = c.createQueryTable("EXPECTED", "SELECT * FROM src_dataset where name='IUCLID5 .i5z file'");
    Assert.assertEquals(1, srcdataset.getRowCount());
    struc_src = c.createQueryTable("EXPECTED", "SELECT * FROM struc_dataset");
    Assert.assertEquals(1, struc_src.getRowCount());
    property = c.createQueryTable("EXPECTED", "SELECT * FROM substance_protocolapplication where topcategory='P-CHEM' and endpointcategory='ASPECT_RATIO_SHAPE_SECTION' and interpretation_result='spherical'");
    Assert.assertEquals(1, property.getRowCount());
    property = c.createQueryTable("EXPECTED", "SELECT * FROM substance_experiment where topcategory='P-CHEM' and endpointcategory='ASPECT_RATIO_SHAPE_SECTION'");
    Assert.assertEquals(4, property.getRowCount());
    c.close();
}

Example 2 with I5ZReader

use of net.idea.i5.io.I5ZReader in project ambit-mirror by ideaconsult.

the class Context method importI5Z.

protected int importI5Z(IStructureKey keytomatch, boolean i6, StructureRecordValidator validator) throws Exception {
    // validator uses parsertype
    setParserType(i6 ? _parsertype.i6z : _parsertype.i5z);
    logger_cli.log(Level.INFO, "MSG_IMPORT", new Object[] { String.format("i%sz", i6 ? "6" : "5"), inputFile.getAbsolutePath() });
    IZReader reader = null;
    Connection c = null;
    try {
        DBConnectionConfigurable<Context> dbc = null;
        dbc = getConnection(getConfigFile());
        c = dbc.getConnection();
        c.setAutoCommit(true);
        I5Options options = new I5Options();
        options.setMaxReferenceStructures(maxRefSubstances);
        options.setExceptionOnMaxReferenceStructures(false);
        options.setAllowMultipleSubstances(false);
        if (i6)
            reader = new I6ZReader<>(inputFile, options);
        else
            reader = new I5ZReader<>(inputFile, options);
        QASettings qa = new QASettings(false);
        qa.setAll();
        reader.setQASettings(qa);
        matchByKey = keytomatch == null ? new CASKey() : keytomatch;
        return write(reader, c, matchByKey, true, clearMeasurements, clearComposition, validator, null, true, false);
    } catch (Exception x) {
        throw x;
    } finally {
        if (reader != null)
            reader.close();
        try {
            if (c != null)
                c.close();
        } catch (Exception x) {
        }
    }
}

Example 3 with I5ZReader

use of net.idea.i5.io.I5ZReader in project ambit-mirror by ideaconsult.

the class CallableSubstanceImporter method createBatch.

@Override
protected AbstractBatchProcessor createBatch(FileInputState target) throws Exception {
    if (target == null)
        throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST);
    final BatchDBProcessor<String> batch = new BatchDBProcessor<String>() {

        /**
         */
        private static final long serialVersionUID = 5712170806359764006L;

        @Override
        public Iterator<String> getIterator(IInputState target) throws AmbitException {
            try {
                IRawReader<IStructureRecord> reader = null;
                File file = ((FileInputState) target).getFile();
                String ext = file.getName().toLowerCase();
                if (ext.endsWith(FileInputState._FILE_TYPE.I5Z_INDEX.getExtension())) {
                    if (writer instanceof DBSubstanceWriter)
                        if (writer instanceof DBSubstanceWriter) {
                            ((DBSubstanceWriter) writer).setSplitRecord(true);
                            ((DBSubstanceWriter) writer).setI5mode(true);
                        }
                    reader = new I5ZReader(file);
                    ((I5ZReader) reader).setQASettings(getQASettings());
                } else if (ext.endsWith(FileInputState._FILE_TYPE.I6Z_INDEX.getExtension())) {
                    if (writer instanceof DBSubstanceWriter)
                        if (writer instanceof DBSubstanceWriter) {
                            ((DBSubstanceWriter) writer).setSplitRecord(true);
                            ((DBSubstanceWriter) writer).setI5mode(true);
                        }
                    reader = new I6ZReader(file);
                    ((I6ZReader) reader).setQASettings(getQASettings());
                } else if (ext.endsWith(FileInputState._FILE_TYPE.CSV_INDEX.getExtension())) {
                    if (writer instanceof DBSubstanceWriter)
                        ((DBSubstanceWriter) writer).setSplitRecord(false);
                    LiteratureEntry reference = new LiteratureEntry(originalname, originalname);
                    reader = new CSV12SubstanceReader(new CSV12Reader(new FileReader(file), reference, "FCSV-"));
                } else if (ext.endsWith(".rdf")) {
                    if (writer instanceof DBSubstanceWriter) {
                        ((DBSubstanceWriter) writer).setSplitRecord(false);
                        ((DBSubstanceWriter) writer).setImportBundles(true);
                    }
                    reader = new NanoWikiRDFReader(new InputStreamReader(new FileInputStream(file), "UTF-8"), null, "RDF/XML");
                } else if (ext.endsWith(".ttl")) {
                    if (writer instanceof DBSubstanceWriter) {
                        ((DBSubstanceWriter) writer).setSplitRecord(false);
                        ((DBSubstanceWriter) writer).setImportBundles(true);
                    }
                    reader = new ENanoMapperRDFReader(new InputStreamReader(new FileInputStream(file), "UTF-8"), "ENM3");
                } else if (FileInputState._FILE_TYPE.XLSX_INDEX.hasExtension(ext) || FileInputState._FILE_TYPE.XLS_INDEX.hasExtension(ext)) {
                    if (configFile == null)
                        throw new AmbitException("XLSX/XLSX file import requires a JSON configuration file");
                    final StructureRecordValidator validator = new StructureRecordValidator(file.getName(), true, "XLSX");
                    reader = new GenericExcelParser(new FileInputStream(file), configFile, FileInputState._FILE_TYPE.XLSX_INDEX.hasExtension(ext)) {

                        public Object next() {
                            Object record = super.next();
                            try {
                                if (record instanceof IStructureRecord)
                                    record = validator.process((IStructureRecord) record);
                            } catch (Exception x) {
                            }
                            return record;
                        }
                    };
                    if (writer instanceof DBSubstanceWriter) {
                        ((DBSubstanceWriter) writer).setSplitRecord(false);
                    /*
							 * ((DBSubstanceWriter) writer)
							 * .setClearComposition(false); ((DBSubstanceWriter)
							 * writer) .setClearMeasurements(false);
							 */
                    }
                } else if (ext.endsWith(".json")) {
                    if (writer instanceof DBSubstanceWriter)
                        ((DBSubstanceWriter) writer).setSplitRecord(false);
                    reader = new SubstanceStudyParser(new InputStreamReader(new FileInputStream(file), "UTF-8")) {

                        protected EffectRecord createEffectRecord(Protocol protocol) {
                            try {
                                I5_ROOT_OBJECTS category = I5_ROOT_OBJECTS.valueOf(protocol.getCategory() + "_SECTION");
                                return category.createEffectRecord();
                            } catch (Exception x) {
                                return super.createEffectRecord(protocol);
                            }
                        }
                    };
                    if (writer instanceof DBSubstanceWriter) {
                        ((DBSubstanceWriter) writer).setClearComposition(false);
                        ((DBSubstanceWriter) writer).setClearMeasurements(false);
                    }
                } else {
                    throw new AmbitException("Unsupported format " + file);
                }
                reader.setErrorHandler(new IChemObjectReaderErrorHandler() {

                    @Override
                    public void handleError(String message, int row, int colStart, int colEnd, Exception exception) {
                    }

                    @Override
                    public void handleError(String message, int row, int colStart, int colEnd) {
                    }

                    @Override
                    public void handleError(String message, Exception exception) {
                    }

                    @Override
                    public void handleError(String message) {
                    }
                });
                return reader;
            } catch (AmbitException x) {
                throw x;
            } catch (Exception x) {
                throw new AmbitException(x);
            }
        }
    };
    return batch;
}