Examples with ItfReader ch.interlis.iom_j.itf.ItfReader used on opensource projects

Example 1 with ItfReader

use of ch.interlis.iom_j.itf.ItfReader in project ili2db by claeis.

the class TransferFromXtf method doit.

public void doit(IoxReader reader, Config config, Map<Long, BasketStat> stat) throws IoxException, Ili2dbException {
    if (functionCode == Config.FC_UPDATE || functionCode == Config.FC_REPLACE) {
        if (!createBasketCol) {
            throw new Ili2dbException("update/replace requires a basket column");
        }
    }
    // limit import to given BIDs
    HashSet<String> limitedToBids = null;
    {
        String baskets = config.getBaskets();
        if (baskets != null) {
            String[] basketidv = baskets.split(ch.interlis.ili2c.Main.MODELS_SEPARATOR);
            limitedToBids = new HashSet<String>();
            for (String basketid : basketidv) {
                limitedToBids.add(basketid);
            }
        }
    }
    // limit import to given TOPICs
    HashSet<String> limitedToTopics = null;
    {
        String topics = config.getTopics();
        if (topics != null) {
            String[] topicv = topics.split(ch.interlis.ili2c.Main.MODELS_SEPARATOR);
            limitedToTopics = new HashSet<String>();
            for (String topic : topicv) {
                limitedToTopics.add(topic);
            }
        }
    }
    String datasetName = config.getDatasetName();
    Long datasetSqlId = null;
    today = new java.sql.Timestamp(System.currentTimeMillis());
    if (doItfLineTables) {
        tag2class = ch.interlis.iom_j.itf.ModelUtilities.getTagMap(td);
    } else {
        tag2class = ch.interlis.ili2c.generator.XSDGenerator.getTagMap(td);
    }
    isItfReader = reader instanceof ItfReader;
    unknownTypev = new HashSet();
    structQueue = new ArrayList();
    boolean surfaceAsPolyline = true;
    boolean ignoreUnresolvedReferences = false;
    recman = new ObjectPoolManager();
    try {
        objPool = recman.newObjectPool();
        long importSqlId = 0;
        long basketSqlId = 0;
        long startTid = 0;
        long endTid = 0;
        long objCount = 0;
        boolean referrs = false;
        recConv = new FromXtfRecordConverter(td, ili2sqlName, tag2class, config, idGen, geomConv, conn, dbusr, isItfReader, oidPool, trafoConfig, class2wrapper, datasetName);
        if (functionCode == Config.FC_DELETE || functionCode == Config.FC_REPLACE) {
            if (datasetName == null) {
                throw new Ili2dbException("delete/replace requires a dataset name");
            }
            // delete existing data base on basketSqlId
            datasetSqlId = Ili2db.getDatasetId(datasetName, conn, config);
            if (datasetSqlId == null) {
                if (functionCode == Config.FC_DELETE) {
                // nothing to do
                } else if (functionCode == Config.FC_REPLACE) {
                    // new dataset, not a replace!
                    datasetSqlId = oidPool.newObjSqlId();
                    try {
                        writeDataset(datasetSqlId, datasetName);
                        importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
                    } catch (SQLException e) {
                        EhiLogger.logError(e);
                    } catch (ConverterException e) {
                        EhiLogger.logError(e);
                    }
                }
            } else {
                deleteObjectsOfExistingDataset(datasetSqlId, config);
                if (functionCode == Config.FC_DELETE) {
                    String sqlName = DbNames.DATASETS_TAB;
                    if (schema != null) {
                        sqlName = schema + "." + sqlName;
                    }
                    java.sql.PreparedStatement getstmt = null;
                    try {
                        String stmt = "DELETE FROM " + sqlName + " WHERE " + colT_ID + "= ?";
                        EhiLogger.traceBackendCmd(stmt);
                        getstmt = conn.prepareStatement(stmt);
                        getstmt.setLong(1, datasetSqlId);
                        getstmt.executeUpdate();
                    } catch (java.sql.SQLException ex) {
                        throw new Ili2dbException("failed to delete from " + sqlName, ex);
                    } finally {
                        if (getstmt != null) {
                            try {
                                getstmt.close();
                                getstmt = null;
                            } catch (java.sql.SQLException ex) {
                                EhiLogger.logError(ex);
                            }
                        }
                    }
                }
                try {
                    importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
                } catch (SQLException e) {
                    EhiLogger.logError(e);
                } catch (ConverterException e) {
                    EhiLogger.logError(e);
                }
            }
        } else if (functionCode == Config.FC_UPDATE) {
            if (datasetName == null) {
                throw new Ili2dbException("update requires a dataset name");
            }
            try {
                datasetSqlId = Ili2db.getDatasetId(datasetName, conn, config);
                if (datasetSqlId != null) {
                } else {
                    datasetSqlId = oidPool.newObjSqlId();
                    writeDataset(datasetSqlId, datasetName);
                }
                importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
            } catch (SQLException e) {
                EhiLogger.logError(e);
            } catch (ConverterException e) {
                EhiLogger.logError(e);
            }
        } else if (functionCode == Config.FC_IMPORT) {
            try {
                if (datasetName == null) {
                    datasetSqlId = oidPool.newObjSqlId();
                    if (xtffilename != null) {
                        datasetName = new java.io.File(xtffilename).getName() + "-" + Long.toString(datasetSqlId);
                    } else {
                        datasetName = Long.toString(datasetSqlId);
                    }
                } else {
                    datasetSqlId = Ili2db.getDatasetId(datasetName, conn, config);
                    if (datasetSqlId != null) {
                        throw new Ili2dbException("dataset " + datasetName + " already exists");
                    } else {
                        datasetSqlId = oidPool.newObjSqlId();
                    }
                }
                writeDataset(datasetSqlId, datasetName);
                importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
            } catch (SQLException e) {
                EhiLogger.logError(e);
            } catch (ConverterException e) {
                EhiLogger.logError(e);
            }
        } else {
            throw new IllegalArgumentException("unexpected function code " + functionCode);
        }
        if (functionCode == Config.FC_DELETE) {
            return;
        }
        if (reader instanceof ItfReader) {
            ((ItfReader) reader).setBidPrefix(datasetName);
        } else if (reader instanceof ItfReader2) {
            ((ItfReader2) reader).setBidPrefix(datasetName);
        }
        ch.interlis.iox_j.validator.Validator validator = null;
        if (config.isValidation()) {
            ValidationConfig modelConfig = new ValidationConfig();
            modelConfig.mergeIliMetaAttrs(td);
            String configFilename = config.getValidConfigFile();
            if (configFilename != null) {
                try {
                    modelConfig.mergeConfigFile(new File(configFilename));
                } catch (FileNotFoundException e) {
                    EhiLogger.logError("validator config file <" + configFilename + "> not found");
                }
            }
            modelConfig.setConfigValue(ValidationConfig.PARAMETER, ValidationConfig.AREA_OVERLAP_VALIDATION, config.isDisableAreaValidation() ? ValidationConfig.OFF : null);
            modelConfig.setConfigValue(ValidationConfig.PARAMETER, ValidationConfig.DEFAULT_GEOMETRY_TYPE_VALIDATION, config.isSkipGeometryErrors() ? ValidationConfig.OFF : null);
            modelConfig.setConfigValue(ValidationConfig.PARAMETER, ValidationConfig.ALLOW_ONLY_MULTIPLICITY_REDUCTION, config.isOnlyMultiplicityReduction() ? ValidationConfig.ON : null);
            IoxLogging errHandler = new ch.interlis.iox_j.logging.Log2EhiLogger();
            LogEventFactory errFactory = new LogEventFactory();
            errFactory.setDataSource(xtffilename);
            PipelinePool pipelinePool = new PipelinePool();
            validator = new ch.interlis.iox_j.validator.Validator(td, modelConfig, errHandler, errFactory, pipelinePool, config);
            if (reader instanceof ItfReader2) {
                ((ItfReader2) reader).setIoxDataPool(pipelinePool);
            }
        }
        // setup list of objects that have external/forward references
        delayedObjects = new ArrayList<FixIomObjectExtRefs>();
        HashMap<String, ClassStat> objStat = null;
        StartBasketEvent basket = null;
        // more baskets?
        IoxEvent event = reader.read();
        try {
            boolean skipBasket = false;
            while (event != null) {
                if (event instanceof StartBasketEvent) {
                    basket = (StartBasketEvent) event;
                    // do not import this this basket?
                    if ((limitedToBids != null && !limitedToBids.contains(basket.getBid())) || (limitedToTopics != null && !limitedToTopics.contains(basket.getType()))) {
                        // do not import this basket
                        skipBasket = true;
                        EhiLogger.logState("Skip Basket " + basket.getType() + "(oid " + basket.getBid() + ")");
                        if (validator != null)
                            validator.validate(event);
                        if (languageFilter != null) {
                            event = languageFilter.filter(event);
                        }
                    } else {
                        // import this basket
                        EhiLogger.logState("Basket " + basket.getType() + "(oid " + basket.getBid() + ")...");
                        skipBasket = false;
                        try {
                            if (validator != null)
                                validator.validate(event);
                            if (languageFilter != null) {
                                event = languageFilter.filter(event);
                            }
                            Long existingBasketSqlId = null;
                            if (functionCode == Config.FC_UPDATE) {
                                // read existing oid/sqlid mapping (but might also be a new basket)
                                existingObjectsOfCurrentBasket = new HashMap<String, HashSet<Long>>();
                                existingBasketSqlId = readExistingSqlObjIds(reader instanceof ItfReader, basket.getBid());
                                if (existingBasketSqlId == null) {
                                    // new basket
                                    basketSqlId = oidPool.getBasketSqlId(basket.getBid());
                                } else {
                                    // existing basket
                                    basketSqlId = existingBasketSqlId;
                                    // drop existing structeles
                                    dropExistingStructEles(basket.getType(), basketSqlId);
                                }
                            } else {
                                basketSqlId = oidPool.getBasketSqlId(basket.getBid());
                            }
                            if (attachmentKey == null) {
                                if (xtffilename != null) {
                                    attachmentKey = new java.io.File(xtffilename).getName() + "-" + Long.toString(basketSqlId);
                                } else {
                                    attachmentKey = Long.toString(basketSqlId);
                                }
                                config.setAttachmentKey(attachmentKey);
                            }
                            if (existingBasketSqlId == null) {
                                writeBasket(datasetSqlId, basket, basketSqlId, attachmentKey);
                            } else {
                            // TODO update attachmentKey of existing basket
                            }
                        } catch (SQLException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        } catch (ConverterException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        }
                        startTid = oidPool.getLastSqlId();
                        objStat = new HashMap<String, ClassStat>();
                        objCount = 0;
                    }
                } else if (event instanceof EndBasketEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (languageFilter != null) {
                        event = languageFilter.filter(event);
                    }
                    if (reader instanceof ItfReader2) {
                        ArrayList<IoxInvalidDataException> dataerrs = ((ItfReader2) reader).getDataErrs();
                        if (dataerrs.size() > 0) {
                            if (!skipBasket) {
                                if (!config.isSkipGeometryErrors()) {
                                    for (IoxInvalidDataException dataerr : dataerrs) {
                                        EhiLogger.logError(dataerr);
                                    }
                                }
                            }
                            ((ItfReader2) reader).clearDataErrs();
                        }
                    }
                    if (!skipBasket) {
                        // fix external/forward references
                        ArrayList<FixIomObjectExtRefs> fixedObjects = new ArrayList<FixIomObjectExtRefs>();
                        for (FixIomObjectExtRefs fixref : delayedObjects) {
                            boolean skipObj = false;
                            for (IomObject ref : fixref.getRefs()) {
                                String xtfid = ref.getobjectrefoid();
                                Viewable aclass = fixref.getTargetClass(ref);
                                String rootClassName = Ili2cUtility.getRootViewable(aclass).getScopedName(null);
                                if (oidPool.containsXtfid(rootClassName, xtfid)) {
                                // reference now resolvable
                                } else {
                                    // reference not yet known, try to resolve again at end of transfer
                                    skipObj = true;
                                }
                            }
                            if (!skipObj) {
                                doObject(datasetName, basketSqlId, objPool.get(fixref.getRootTid()), objStat);
                                fixedObjects.add(fixref);
                            }
                        }
                        delayedObjects.removeAll(fixedObjects);
                        if (functionCode == Config.FC_UPDATE) {
                            // delete no longer existing objects
                            deleteExisitingObjects(existingObjectsOfCurrentBasket);
                        }
                        // TODO update import counters
                        endTid = oidPool.getLastSqlId();
                        try {
                            String filename = null;
                            if (xtffilename != null) {
                                filename = new java.io.File(xtffilename).getName();
                            }
                            long importId = writeImportBasketStat(importSqlId, basketSqlId, startTid, endTid, objCount);
                            saveObjStat(stat, importId, basket.getBid(), basketSqlId, filename, basket.getType(), objStat);
                        } catch (SQLException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        } catch (ConverterException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        }
                    }
                    skipBasket = false;
                } else if (event instanceof ObjectEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (!skipBasket) {
                        if (languageFilter != null) {
                            event = languageFilter.filter(event);
                        }
                        objCount++;
                        IomObject iomObj = ((ObjectEvent) event).getIomObject();
                        if (allReferencesKnown(basketSqlId, iomObj)) {
                            // translate object
                            doObject(datasetName, basketSqlId, iomObj, objStat);
                        }
                    }
                } else if (event instanceof EndTransferEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (languageFilter != null) {
                        event = languageFilter.filter(event);
                    }
                    {
                        for (FixIomObjectExtRefs fixref : delayedObjects) {
                            boolean skipObj = false;
                            for (IomObject ref : fixref.getRefs()) {
                                String xtfid = ref.getobjectrefoid();
                                Viewable aclass = fixref.getTargetClass(ref);
                                String rootClassName = Ili2cUtility.getRootViewable(aclass).getScopedName(null);
                                if (oidPool.containsXtfid(rootClassName, xtfid)) {
                                // skip it; now resolvable
                                } else {
                                    // object in another basket
                                    if (readIliTid || (aclass instanceof AbstractClassDef && ((AbstractClassDef) aclass).getOid() != null)) {
                                        // read object
                                        Long sqlid = readObjectSqlid(aclass, xtfid);
                                        if (sqlid == null) {
                                            if (!ignoreUnresolvedReferences) {
                                                EhiLogger.logError("unknown referenced object " + aclass.getScopedName(null) + " TID " + xtfid + " referenced from " + fixref.getRootTag() + " TID " + fixref.getRootTid());
                                                referrs = true;
                                                skipObj = true;
                                            }
                                        }
                                    } else {
                                        if (!ignoreUnresolvedReferences) {
                                            EhiLogger.logError("unknown referenced object " + aclass.getScopedName(null) + " TID " + xtfid + " referenced from " + fixref.getRootTag() + " TID " + fixref.getRootTid());
                                            referrs = true;
                                            skipObj = true;
                                        }
                                    }
                                }
                            }
                            if (!skipObj) {
                                objStat = stat.get(fixref.getBasketSqlId()).getObjStat();
                                doObject(datasetName, fixref.getBasketSqlId(), objPool.get(fixref.getRootTid()), objStat);
                            }
                        }
                    }
                    break;
                } else if (event instanceof StartTransferEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (languageFilter != null) {
                        event = languageFilter.filter(event);
                    }
                }
                event = reader.read();
            }
            if (referrs) {
                throw new IoxException("dangling references");
            }
        } finally {
            if (validator != null) {
                validator.close();
                validator = null;
            }
            if (languageFilter != null) {
                languageFilter.close();
            }
        }
    } finally {
        recman.close();
    }
}

Example 2 with ItfReader

use of ch.interlis.iom_j.itf.ItfReader in project ili2db by claeis.

the class Datatypes10Test method exportItfWithSkipPolygonBuilding.

@Test
public void exportItfWithSkipPolygonBuilding() throws Exception {
    Connection jdbcConnection = null;
    try {
        Class driverClass = Class.forName("org.postgresql.Driver");
        jdbcConnection = DriverManager.getConnection(dburl, dbuser, dbpwd);
        stmt = jdbcConnection.createStatement();
        stmt.execute("DROP SCHEMA IF EXISTS " + DBSCHEMA + " CASCADE");
        DbUtility.executeSqlScript(jdbcConnection, new java.io.FileReader("test/data/Datatypes10/CreateTable.sql"));
        DbUtility.executeSqlScript(jdbcConnection, new java.io.FileReader("test/data/Datatypes10/InsertIntoTable.sql"));
        File data = new File("test/data/Datatypes10/Datatypes10a-ltout.itf");
        Config config = initConfig(data.getPath(), DBSCHEMA, data.getPath() + ".log");
        config.setModels("Datatypes10");
        config.setFunction(Config.FC_EXPORT);
        Ili2db.readSettingsFromDb(config);
        Ili2db.run(config, null);
        // read objects of db and write objectValue to HashMap
        HashMap<String, IomObject> objs = new HashMap<String, IomObject>();
        ItfReader reader = new ItfReader(data);
        IoxEvent event = null;
        do {
            event = reader.read();
            if (event instanceof StartTransferEvent) {
            } else if (event instanceof StartBasketEvent) {
            } else if (event instanceof ObjectEvent) {
                IomObject iomObj = ((ObjectEvent) event).getIomObject();
                if (iomObj.getobjectoid() != null) {
                    objs.put(iomObj.getobjectoid(), iomObj);
                }
            } else if (event instanceof EndBasketEvent) {
            } else if (event instanceof EndTransferEvent) {
            }
        } while (!(event instanceof EndTransferEvent));
        {
            IomObject obj0 = objs.get("10");
            Assert.assertNotNull(obj0);
            Assert.assertEquals("Datatypes10.Topic.OtherTable", obj0.getobjecttag());
        }
        {
            IomObject obj1 = objs.get("11");
            Assert.assertNotNull(obj1);
            Assert.assertEquals("Datatypes10.Topic.OtherTable", obj1.getobjecttag());
        }
    } finally {
        if (jdbcConnection != null) {
            jdbcConnection.close();
        }
    }
}

Example 3 with ItfReader

use of ch.interlis.iom_j.itf.ItfReader in project ili2db by claeis.

the class Datatypes10Test method exportItf.

@Test
public void exportItf() throws Exception {
    Connection jdbcConnection = null;
    try {
        Class driverClass = Class.forName("org.postgresql.Driver");
        jdbcConnection = DriverManager.getConnection(dburl, dbuser, dbpwd);
        stmt = jdbcConnection.createStatement();
        stmt.execute("DROP SCHEMA IF EXISTS " + DBSCHEMA + " CASCADE");
        DbUtility.executeSqlScript(jdbcConnection, new java.io.FileReader("test/data/Datatypes10/CreateTable.sql"));
        DbUtility.executeSqlScript(jdbcConnection, new java.io.FileReader("test/data/Datatypes10/InsertIntoTable.sql"));
        File data = new File("test/data/Datatypes10/Datatypes10a-out.itf");
        Config config = initConfig(data.getPath(), DBSCHEMA, data.getPath() + ".log");
        config.setModels("Datatypes10");
        config.setFunction(Config.FC_EXPORT);
        Ili2db.readSettingsFromDb(config);
        Ili2db.run(config, null);
        // read objects of db and write objectValue to HashMap
        HashMap<String, IomObject> objs = new HashMap<String, IomObject>();
        ItfReader reader = new ItfReader(data);
        IoxEvent event = null;
        do {
            event = reader.read();
            if (event instanceof StartTransferEvent) {
            } else if (event instanceof StartBasketEvent) {
            } else if (event instanceof ObjectEvent) {
                IomObject iomObj = ((ObjectEvent) event).getIomObject();
                if (iomObj.getobjectoid() != null) {
                    objs.put(iomObj.getobjectoid(), iomObj);
                }
            } else if (event instanceof EndBasketEvent) {
            } else if (event instanceof EndTransferEvent) {
            }
        } while (!(event instanceof EndTransferEvent));
        {
            IomObject obj0 = objs.get("10");
            Assert.assertNotNull(obj0);
            Assert.assertEquals("Datatypes10.Topic.OtherTable", obj0.getobjecttag());
        }
        {
            IomObject obj1 = objs.get("11");
            Assert.assertNotNull(obj1);
            Assert.assertEquals("Datatypes10.Topic.OtherTable", obj1.getobjecttag());
        }
    } finally {
        if (jdbcConnection != null) {
            jdbcConnection.close();
        }
    }
}

Example 4 with ItfReader

use of ch.interlis.iom_j.itf.ItfReader in project ili2db by claeis.

the class Ili2db method runUpdate.

public static void runUpdate(Config config, String appHome, int function) throws Ili2dbException {
    ch.ehi.basics.logging.FileListener logfile = null;
    if (config.getLogfile() != null) {
        logfile = new FileLogger(new java.io.File(config.getLogfile()));
        EhiLogger.getInstance().addListener(logfile);
    }
    StdLogger logStderr = new StdLogger(config.getLogfile());
    EhiLogger.getInstance().addListener(logStderr);
    EhiLogger.getInstance().removeListener(StdListener.getInstance());
    try {
        boolean connectionFromExtern = config.getJdbcConnection() != null;
        logGeneralInfo(config);
        // String zipfilename=null;
        java.util.zip.ZipEntry zipXtfEntry = null;
        java.util.zip.ZipFile zipFile = null;
        String inputFilename = config.getXtffile();
        if (function == Config.FC_DELETE) {
            if (config.getDatasetName() == null) {
                throw new Ili2dbException("no datasetName given");
            }
        } else {
            if (inputFilename == null) {
                throw new Ili2dbException("no xtf-file given");
            }
            if (ch.ehi.basics.view.GenericFileFilter.getFileExtension(inputFilename).toLowerCase().equals("zip")) {
                try {
                    zipFile = new java.util.zip.ZipFile(inputFilename);
                } catch (IOException ex) {
                    throw new Ili2dbException(ex);
                }
                java.util.Enumeration filei = zipFile.entries();
                while (filei.hasMoreElements()) {
                    java.util.zip.ZipEntry zipEntry = (java.util.zip.ZipEntry) filei.nextElement();
                    String ext = ch.ehi.basics.view.GenericFileFilter.getFileExtension(zipEntry.getName()).toLowerCase();
                    if (ext != null && (ext.equals("xml") || ext.equals("xtf") || ext.equals("itf"))) {
                        zipXtfEntry = zipEntry;
                        break;
                    }
                }
                if (zipXtfEntry == null) {
                    throw new Ili2dbException("no xtf/itf-file in zip-archive " + zipFile.getName());
                }
            }
        }
        String modeldir = config.getModeldir();
        if (modeldir == null) {
            throw new Ili2dbException("no modeldir given");
        }
        EhiLogger.traceState("modeldir <" + modeldir + ">");
        ch.interlis.ili2c.config.Configuration modelv = new ch.interlis.ili2c.config.Configuration();
        if (function != Config.FC_DELETE) {
            String models = config.getModels();
            if (models == null) {
                throw new Ili2dbException("no models given");
            }
            EhiLogger.traceState("models <" + models + ">");
            String[] modelnames = getModelNames(models);
            for (int modeli = 0; modeli < modelnames.length; modeli++) {
                String m = modelnames[modeli];
                if (m.equals(XTF)) {
                    // read modelname from xtf-file
                    if (zipXtfEntry != null) {
                        try {
                            java.io.InputStream in = zipFile.getInputStream(zipXtfEntry);
                            m = getModelFromXtf(in, zipXtfEntry.getName());
                        } catch (java.io.IOException ex) {
                            throw new Ili2dbException(ex);
                        }
                    } else {
                        m = getModelFromXtf(inputFilename);
                    }
                }
                if (m != null) {
                    modelv.addFileEntry(new ch.interlis.ili2c.config.FileEntry(m, ch.interlis.ili2c.config.FileEntryKind.ILIMODELFILE));
                }
            }
        }
        String dburl = config.getDburl();
        String dbusr = config.getDbusr();
        String dbpwd = config.getDbpwd();
        if (!connectionFromExtern && dburl == null) {
            throw new Ili2dbException("no dburl given");
        }
        if (dbusr == null) {
            // EhiLogger.logError("no dbusr given");
            // return;
            dbusr = "";
        }
        if (dbpwd == null) {
            // EhiLogger.logError("no dbpwd given");
            // return;
            dbpwd = "";
        }
        String dbschema = config.getDbschema();
        if (dbschema != null) {
            EhiLogger.logState("dbschema <" + dbschema + ">");
        }
        String geometryConverter = config.getGeometryConverter();
        if (geometryConverter == null) {
            throw new Ili2dbException("no geoemtry converter given");
        }
        String ddlGenerator = config.getDdlGenerator();
        if (ddlGenerator == null) {
            throw new Ili2dbException("no DDL generator given");
        }
        String idGenerator = config.getIdGenerator();
        if (idGenerator == null) {
            throw new Ili2dbException("no ID generator given");
        }
        if (!connectionFromExtern) {
            String jdbcDriver = config.getJdbcDriver();
            if (jdbcDriver == null) {
                throw new Ili2dbException("no JDBC driver given");
            }
            try {
                Class.forName(jdbcDriver);
            } catch (Exception ex) {
                throw new Ili2dbException("failed to load JDBC driver", ex);
            }
        }
        CustomMapping customMapping = getCustomMappingStrategy(config);
        // open db connection
        Connection conn = null;
        String url = dburl;
        ch.ehi.basics.logging.ErrorTracker errs = null;
        try {
            try {
                if (connectionFromExtern) {
                    conn = config.getJdbcConnection();
                } else {
                    conn = connect(url, dbusr, dbpwd, config, customMapping);
                }
                customMapping.postConnect(conn, config);
            } catch (SQLException ex) {
                throw new Ili2dbException("failed to get db connection", ex);
            }
            logDBVersion(conn);
            if (!connectionFromExtern) {
                // switch off auto-commit
                try {
                    conn.setAutoCommit(false);
                } catch (SQLException ex) {
                    throw new Ili2dbException("failed to switch off auto-commit", ex);
                }
            }
            // run pre-script
            if (config.getPreScript() != null) {
                try {
                    DbUtility.executeSqlScript(conn, new java.io.FileReader(config.getPreScript()));
                    EhiLogger.logState("run update pre-script...");
                } catch (FileNotFoundException e) {
                    throw new Ili2dbException("update pre-script statements failed", e);
                }
            }
            // create db schema
            if (function == Config.FC_IMPORT) {
                if (config.getDbschema() != null) {
                    if (!DbUtility.schemaExists(conn, config.getDbschema())) {
                        DbUtility.createSchema(conn, config.getDbschema());
                    }
                }
            }
            if (function == Config.FC_DELETE) {
                boolean createBasketCol = config.BASKET_HANDLING_READWRITE.equals(config.getBasketHandling());
                if (!createBasketCol) {
                    throw new Ili2dbException("delete requires column " + DbNames.T_BASKET_COL);
                }
                String datasetName = config.getDatasetName();
                // map datasetName to modelnames
                Long datasetId = getDatasetId(datasetName, conn, config);
                if (datasetId == null) {
                    throw new Ili2dbException("dataset <" + datasetName + "> doesn't exist");
                }
                getBasketSqlIdsFromDatasetId(datasetId, modelv, conn, config);
            }
            if (function == Config.FC_IMPORT) {
                String datasetName = config.getDatasetName();
                if (datasetName != null) {
                    if (DbUtility.tableExists(conn, new DbTableName(config.getDbschema(), DbNames.DATASETS_TAB))) {
                        Long datasetId = getDatasetId(datasetName, conn, config);
                        if (datasetId != null) {
                            throw new Ili2dbException("dataset <" + datasetName + "> already exists");
                        }
                    }
                    boolean createBasketCol = config.BASKET_HANDLING_READWRITE.equals(config.getBasketHandling());
                    if (!createBasketCol) {
                        throw new Ili2dbException("import with dataset name requires column " + DbNames.T_BASKET_COL);
                    }
                }
            }
            if (modelv.getSizeFileEntry() == 0) {
                throw new Ili2dbException("no models given");
            }
            // compile required ili files
            setupIli2cPathmap(config, appHome, inputFilename, conn);
            Ili2cMetaAttrs ili2cMetaAttrs = new Ili2cMetaAttrs();
            ch.interlis.ili2c.config.Configuration ili2cConfig = null;
            try {
                ili2cConfig = (ch.interlis.ili2c.config.Configuration) modelv.clone();
            } catch (CloneNotSupportedException e1) {
                throw new Ili2dbException(e1);
            }
            setupIli2cMetaAttrs(ili2cMetaAttrs, config, ili2cConfig);
            EhiLogger.logState("compile models...");
            ili2cConfig.setAutoCompleteModelList(true);
            ili2cConfig.setGenerateWarnings(false);
            TransferDescription td = ch.interlis.ili2c.Main.runCompiler(ili2cConfig, config, ili2cMetaAttrs);
            if (td == null) {
                throw new Ili2dbException("compiler failed");
            }
            // read mapping file
            NameMapping mapping = new NameMapping(config);
            if (DbUtility.tableExists(conn, new DbTableName(config.getDbschema(), DbNames.CLASSNAME_TAB))) {
                // read mapping from db
                mapping.readTableMappingTable(conn, config.getDbschema());
            }
            if (DbUtility.tableExists(conn, new DbTableName(config.getDbschema(), DbNames.ATTRNAME_TAB))) {
                // read mapping from db
                mapping.readAttrMappingTable(conn, config.getDbschema());
            }
            TrafoConfig trafoConfig = new TrafoConfig();
            trafoConfig.readTrafoConfig(conn, config.getDbschema());
            ModelElementSelector ms = new ModelElementSelector();
            ArrayList<String> modelNames = new ArrayList<String>();
            for (int modeli = 0; modeli < modelv.getSizeFileEntry(); modeli++) {
                if (modelv.getFileEntry(modeli).getKind() == ch.interlis.ili2c.config.FileEntryKind.ILIMODELFILE) {
                    String m = modelv.getFileEntry(modeli).getFilename();
                    modelNames.add(m);
                }
            }
            // use models explicitly given by user --models, --topics and/or as read from transferfile
            java.util.List<Element> eles = ms.getModelElements(modelNames, td, td.getIli1Format() != null && config.getDoItfLineTables(), Config.CREATE_ENUM_DEFS_MULTI.equals(config.getCreateEnumDefs()), config);
            Viewable2TableMapping class2wrapper = Viewable2TableMapper.getClass2TableMapping(config, trafoConfig, eles, mapping);
            Generator gen = null;
            try {
                gen = (Generator) Class.forName(ddlGenerator).newInstance();
            } catch (Exception ex) {
                throw new Ili2dbException("failed to load/create DDL generator", ex);
            }
            DbIdGen idGen = null;
            try {
                idGen = (DbIdGen) Class.forName(idGenerator).newInstance();
            } catch (Exception ex) {
                throw new Ili2dbException("failed to load/create ID generator", ex);
            }
            idGen.init(config.getDbschema(), config);
            SqlColumnConverter geomConverter = null;
            try {
                geomConverter = (SqlColumnConverter) Class.forName(geometryConverter).newInstance();
            } catch (Exception ex) {
                throw new Ili2dbException("failed to load/create geometry converter", ex);
            }
            geomConverter.setup(conn, config);
            idGen.initDb(conn, dbusr);
            idGen.initDbDefs(gen);
            // create table structure
            if (function == Config.FC_IMPORT) {
                EhiLogger.logState("create table structure...");
                try {
                    TransferFromIli trsfFromIli = new TransferFromIli();
                    // map ili-classes to sql-tables
                    DbSchema schema;
                    try {
                        schema = trsfFromIli.doit(td, eles, mapping, config, idGen, trafoConfig, class2wrapper, customMapping);
                    } catch (Ili2dbException e) {
                        throw new Ili2dbException("mapping of ili-classes to sql-tables failed", e);
                    }
                    if (schema == null) {
                        return;
                    }
                    trsfFromIli.addBasketsTable(schema);
                    trsfFromIli.addImportsTable(schema);
                    TransferFromIli.addInheritanceTable(schema, Integer.parseInt(config.getMaxSqlNameLength()));
                    TransferFromIli.addSettingsTable(schema);
                    TransferFromIli.addTrafoConfigTable(schema);
                    TransferFromIli.addModelsTable(schema, config);
                    trsfFromIli.addEnumTable(schema);
                    TransferFromIli.addTableMappingTable(schema);
                    TransferFromIli.addAttrMappingTable(schema);
                    DbExtMetaInfo.addMetaInfoTables(schema);
                    idGen.addMappingTable(schema);
                    if (config.getCreateMetaInfo()) {
                        MetaAttrUtility.addMetaAttributesTable(schema);
                    }
                    GeneratorDriver drv = new GeneratorDriver(gen);
                    drv.visitSchema(config, schema);
                    // create script requested by user?
                    String createscript = config.getCreatescript();
                    if (createscript != null && (gen instanceof GeneratorJdbc)) {
                        writeScript(createscript, ((GeneratorJdbc) gen).iteratorCreateLines());
                    }
                    // drop script requested by user?
                    String dropscript = config.getDropscript();
                    if (dropscript != null && (gen instanceof GeneratorJdbc)) {
                        writeScript(dropscript, ((GeneratorJdbc) gen).iteratorDropLines());
                    }
                    // update mapping table
                    mapping.updateTableMappingTable(conn, config.getDbschema());
                    mapping.updateAttrMappingTable(conn, config.getDbschema());
                    trafoConfig.updateTrafoConfig(conn, config.getDbschema());
                    // update inheritance table
                    trsfFromIli.updateInheritanceTable(conn, config.getDbschema());
                    // update enumerations table
                    trsfFromIli.updateEnumTable(conn);
                    trsfFromIli.updateMetaInfoTables(conn);
                    TransferFromIli.addModels(conn, td, config.getDbschema());
                    if (!config.isConfigReadFromDb()) {
                        TransferFromIli.updateSettings(conn, config, config.getDbschema());
                    }
                } catch (java.io.IOException ex) {
                    throw new Ili2dbException(ex);
                }
            }
            // process xtf files
            EhiLogger.logState("process data file...");
            Map<Long, BasketStat> stat = new HashMap<Long, BasketStat>();
            errs = new ch.ehi.basics.logging.ErrorTracker();
            EhiLogger.getInstance().addListener(errs);
            if (zipXtfEntry != null) {
                IoxReader ioxReader = null;
                java.io.InputStream in = null;
                try {
                    EhiLogger.logState("data <" + inputFilename + ":" + zipXtfEntry.getName() + ">");
                    in = zipFile.getInputStream(zipXtfEntry);
                    if (isItfFilename(zipXtfEntry.getName())) {
                        if (config.getDoItfLineTables()) {
                            ioxReader = new ItfReader(in);
                            ((ItfReader) ioxReader).setModel(td);
                        } else {
                            ioxReader = new ItfReader2(in, config.isSkipGeometryErrors());
                            ((ItfReader2) ioxReader).setModel(td);
                        }
                    } else {
                        ioxReader = new XtfReader(in);
                    }
                    transferFromXtf(conn, ioxReader, function, mapping, td, dbusr, geomConverter, idGen, config, stat, trafoConfig, class2wrapper);
                } catch (IOException ex) {
                    throw new Ili2dbException(ex);
                } catch (IoxException ex) {
                    throw new Ili2dbException(ex);
                } finally {
                    if (ioxReader != null) {
                        try {
                            ioxReader.close();
                        } catch (IoxException e) {
                            throw new Ili2dbException(e);
                        }
                        ioxReader = null;
                    }
                    if (in != null) {
                        try {
                            in.close();
                        } catch (IOException e) {
                            throw new Ili2dbException(e);
                        }
                        in = null;
                    }
                }
                // save attachments
                String attachmentKey = config.getAttachmentKey();
                String attachmentsBase = config.getAttachmentsPath();
                if (attachmentsBase != null) {
                    java.io.File basePath = new java.io.File(attachmentsBase, attachmentKey);
                    java.util.Enumeration filei = zipFile.entries();
                    while (filei.hasMoreElements()) {
                        java.util.zip.ZipEntry zipEntry = (java.util.zip.ZipEntry) filei.nextElement();
                        if (!zipXtfEntry.getName().equals(zipEntry.getName())) {
                            // save file
                            java.io.File destFile = new java.io.File(basePath, zipEntry.getName());
                            java.io.File parent = destFile.getAbsoluteFile().getParentFile();
                            if (!parent.exists()) {
                                if (!parent.mkdirs()) {
                                    throw new Ili2dbException("failed to create " + parent.getAbsolutePath());
                                }
                            }
                            try {
                                copyStream(destFile, zipFile.getInputStream(zipEntry));
                            } catch (IOException ex) {
                                throw new Ili2dbException("failed to save attachment " + zipEntry.getName(), ex);
                            }
                        }
                    }
                }
            } else {
                IoxReader ioxReader = null;
                try {
                    if (function != config.FC_DELETE) {
                        EhiLogger.logState("data <" + inputFilename + ">");
                        if (isItfFilename(inputFilename)) {
                            config.setValue(ch.interlis.iox_j.validator.Validator.CONFIG_DO_ITF_OIDPERTABLE, ch.interlis.iox_j.validator.Validator.CONFIG_DO_ITF_OIDPERTABLE_DO);
                            if (config.getDoItfLineTables()) {
                                ioxReader = new ItfReader(new java.io.File(inputFilename));
                                ((ItfReader) ioxReader).setModel(td);
                                ((ItfReader) ioxReader).setBidPrefix(config.getDatasetName());
                            } else {
                                ioxReader = new ItfReader2(new java.io.File(inputFilename), config.isSkipGeometryErrors());
                                ((ItfReader2) ioxReader).setModel(td);
                                ((ItfReader2) ioxReader).setBidPrefix(config.getDatasetName());
                            }
                        } else {
                            ioxReader = new XtfReader(new java.io.File(inputFilename));
                        }
                    }
                    transferFromXtf(conn, ioxReader, function, mapping, td, dbusr, geomConverter, idGen, config, stat, trafoConfig, class2wrapper);
                } catch (IoxException e) {
                    throw new Ili2dbException(e);
                } finally {
                    if (ioxReader != null) {
                        try {
                            ioxReader.close();
                        } catch (IoxException e) {
                            throw new Ili2dbException(e);
                        }
                        ioxReader = null;
                    }
                }
            }
            // import meta-attributes from .toml file
            if (config.getIliMetaAttrsFile() != null) {
                if (config.getCreateMetaInfo()) {
                    try {
                        EhiLogger.logState("run import meta-attributes from toml file");
                        MetaAttrUtility.addMetaAttrsFromToml(td, new java.io.FileReader(config.getIliMetaAttrsFile()));
                    } catch (FileNotFoundException e) {
                        throw new Ili2dbException("import meta-attributes failed", e);
                    }
                } else {
                    throw new Ili2dbException("import meta-attributes requires --createMetaInfo option");
                }
            }
            if (config.getCreateMetaInfo()) {
                // update meta-attributes table
                MetaAttrUtility.updateMetaAttributesTable(conn, config.getDbschema(), td);
                // set elements' meta-attributes
                MetaAttrUtility.addMetaAttrsFromDb(td, conn, config.getDbschema());
            }
            // run post-script
            if (config.getPostScript() != null) {
                try {
                    DbUtility.executeSqlScript(conn, new java.io.FileReader(config.getPostScript()));
                    EhiLogger.logState("run update post-script...");
                } catch (FileNotFoundException e) {
                    throw new Ili2dbException("update post-script statements failed", e);
                }
            }
            if (errs.hasSeenErrors()) {
                if (!connectionFromExtern) {
                    try {
                        conn.rollback();
                    } catch (SQLException e) {
                        EhiLogger.logError("rollback failed", e);
                    }
                }
                throw new Ili2dbException("...import failed");
            } else {
                if (!connectionFromExtern) {
                    try {
                        conn.commit();
                    } catch (SQLException e) {
                        EhiLogger.logError("commit failed", e);
                        throw new Ili2dbException("...import failed");
                    }
                }
                logStatistics(td.getIli1Format() != null, stat);
                EhiLogger.logState("...import done");
            }
        } finally {
            if (!connectionFromExtern) {
                if (conn != null) {
                    try {
                        conn.close();
                    } catch (java.sql.SQLException ex) {
                        EhiLogger.logError(ex);
                    } finally {
                        config.setJdbcConnection(null);
                        conn = null;
                    }
                }
            }
            if (errs != null) {
                EhiLogger.getInstance().removeListener(errs);
                errs = null;
            }
        }
    } catch (Ili2dbException ex) {
        if (logfile != null) {
            logfile.logEvent(new StdLogEvent(LogEvent.ERROR, null, ex, null));
        }
        throw ex;
    } catch (java.lang.RuntimeException ex) {
        if (logfile != null) {
            logfile.logEvent(new StdLogEvent(LogEvent.ERROR, null, ex, null));
        }
        throw ex;
    } finally {
        if (logfile != null) {
            EhiLogger.getInstance().removeListener(logfile);
            logfile.close();
            logfile = null;
        }
        if (logStderr != null) {
            EhiLogger.getInstance().addListener(StdListener.getInstance());
            EhiLogger.getInstance().removeListener(logStderr);
        }
    }
}

Example 5 with ItfReader

use of ch.interlis.iom_j.itf.ItfReader in project ili2db by claeis.

the class TranslationTest method exportItf10.

@Test
public void exportItf10() throws Exception {
    Connection jdbcConnection = null;
    try {
        Class driverClass = Class.forName("org.postgresql.Driver");
        jdbcConnection = DriverManager.getConnection(dburl, dbuser, dbpwd);
        stmt = jdbcConnection.createStatement();
        stmt.execute("DROP SCHEMA IF EXISTS " + DBSCHEMA + " CASCADE");
        DbUtility.executeSqlScript(jdbcConnection, new java.io.FileReader("test/data/Translation/CreateTableItf10.sql"));
        DbUtility.executeSqlScript(jdbcConnection, new java.io.FileReader("test/data/Translation/InsertIntoTableItf10.sql"));
        {
            File data = new File("test/data/Translation/ModelAsimple10a-out.itf");
            Config config = initConfig(data.getPath(), DBSCHEMA, data.getPath() + ".log");
            config.setFunction(Config.FC_EXPORT);
            config.setDatasetName("ModelAsimple10");
            Ili2db.readSettingsFromDb(config);
            Ili2db.run(config, null);
            data = new File("test/data/Translation/ModelBsimple10a-out.itf");
            config = initConfig(data.getPath(), DBSCHEMA, data.getPath() + ".log");
            config.setFunction(Config.FC_EXPORT);
            config.setDatasetName("ModelBsimple10");
            Ili2db.readSettingsFromDb(config);
            Ili2db.run(config, null);
            HashMap<String, IomObject> objs = new HashMap<String, IomObject>();
            ItfReader reader = new ItfReader(data);
            IoxEvent event = null;
            do {
                event = reader.read();
                if (event instanceof StartTransferEvent) {
                } else if (event instanceof StartBasketEvent) {
                } else if (event instanceof ObjectEvent) {
                    IomObject iomObj = ((ObjectEvent) event).getIomObject();
                    if (iomObj.getobjectoid() != null) {
                        objs.put(iomObj.getobjectoid(), iomObj);
                    }
                } else if (event instanceof EndBasketEvent) {
                } else if (event instanceof EndTransferEvent) {
                }
            } while (!(event instanceof EndTransferEvent));
            {
                IomObject obj0 = objs.get("21");
                Assert.assertNotNull(obj0);
                Assert.assertEquals("ModelBsimple10.TopicB.ClassB", obj0.getobjecttag());
            }
            {
                IomObject obj0 = objs.get("20");
                Assert.assertNotNull(obj0);
                Assert.assertEquals("ModelBsimple10.TopicB.ClassB", obj0.getobjecttag());
            }
            {
                IomObject obj0 = objs.get("22");
                Assert.assertNotNull(obj0);
                Assert.assertEquals("ModelBsimple10.TopicB.ClassB2", obj0.getobjecttag());
            }
            {
                IomObject obj0 = objs.get("25");
                Assert.assertNotNull(obj0);
                Assert.assertEquals("ModelBsimple10.TopicB.ClassB3", obj0.getobjecttag());
            }
            {
                IomObject obj0 = objs.get("26");
                Assert.assertNotNull(obj0);
                Assert.assertEquals("ModelBsimple10.TopicB.ClassB2_geomB", obj0.getobjecttag());
            }
            {
                IomObject obj0 = objs.get("27");
                Assert.assertNotNull(obj0);
                Assert.assertEquals("ModelBsimple10.TopicB.ClassB3_geomB", obj0.getobjecttag());
            }
        }
    } finally {
        if (jdbcConnection != null) {
            jdbcConnection.close();
        }
    }
}