Examples with ObjectPoolManager ch.ehi.iox.objpool.ObjectPoolManager used on opensource projects

Example 1 with ObjectPoolManager

use of ch.ehi.iox.objpool.ObjectPoolManager in project ili2db by claeis.

the class TransferFromXtf method doit.

public void doit(IoxReader reader, Config config, Map<Long, BasketStat> stat) throws IoxException, Ili2dbException {
    if (functionCode == Config.FC_UPDATE || functionCode == Config.FC_REPLACE) {
        if (!createBasketCol) {
            throw new Ili2dbException("update/replace requires a basket column");
        }
    }
    // limit import to given BIDs
    HashSet<String> limitedToBids = null;
    {
        String baskets = config.getBaskets();
        if (baskets != null) {
            String[] basketidv = baskets.split(ch.interlis.ili2c.Main.MODELS_SEPARATOR);
            limitedToBids = new HashSet<String>();
            for (String basketid : basketidv) {
                limitedToBids.add(basketid);
            }
        }
    }
    // limit import to given TOPICs
    HashSet<String> limitedToTopics = null;
    {
        String topics = config.getTopics();
        if (topics != null) {
            String[] topicv = topics.split(ch.interlis.ili2c.Main.MODELS_SEPARATOR);
            limitedToTopics = new HashSet<String>();
            for (String topic : topicv) {
                limitedToTopics.add(topic);
            }
        }
    }
    String datasetName = config.getDatasetName();
    Long datasetSqlId = null;
    today = new java.sql.Timestamp(System.currentTimeMillis());
    if (doItfLineTables) {
        tag2class = ch.interlis.iom_j.itf.ModelUtilities.getTagMap(td);
    } else {
        tag2class = ch.interlis.ili2c.generator.XSDGenerator.getTagMap(td);
    }
    isItfReader = reader instanceof ItfReader;
    unknownTypev = new HashSet();
    structQueue = new ArrayList();
    boolean surfaceAsPolyline = true;
    boolean ignoreUnresolvedReferences = false;
    recman = new ObjectPoolManager();
    try {
        objPool = recman.newObjectPool();
        long importSqlId = 0;
        long basketSqlId = 0;
        long startTid = 0;
        long endTid = 0;
        long objCount = 0;
        boolean referrs = false;
        recConv = new FromXtfRecordConverter(td, ili2sqlName, tag2class, config, idGen, geomConv, conn, dbusr, isItfReader, oidPool, trafoConfig, class2wrapper, datasetName);
        if (functionCode == Config.FC_DELETE || functionCode == Config.FC_REPLACE) {
            if (datasetName == null) {
                throw new Ili2dbException("delete/replace requires a dataset name");
            }
            // delete existing data base on basketSqlId
            datasetSqlId = Ili2db.getDatasetId(datasetName, conn, config);
            if (datasetSqlId == null) {
                if (functionCode == Config.FC_DELETE) {
                // nothing to do
                } else if (functionCode == Config.FC_REPLACE) {
                    // new dataset, not a replace!
                    datasetSqlId = oidPool.newObjSqlId();
                    try {
                        writeDataset(datasetSqlId, datasetName);
                        importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
                    } catch (SQLException e) {
                        EhiLogger.logError(e);
                    } catch (ConverterException e) {
                        EhiLogger.logError(e);
                    }
                }
            } else {
                deleteObjectsOfExistingDataset(datasetSqlId, config);
                if (functionCode == Config.FC_DELETE) {
                    String sqlName = DbNames.DATASETS_TAB;
                    if (schema != null) {
                        sqlName = schema + "." + sqlName;
                    }
                    java.sql.PreparedStatement getstmt = null;
                    try {
                        String stmt = "DELETE FROM " + sqlName + " WHERE " + colT_ID + "= ?";
                        EhiLogger.traceBackendCmd(stmt);
                        getstmt = conn.prepareStatement(stmt);
                        getstmt.setLong(1, datasetSqlId);
                        getstmt.executeUpdate();
                    } catch (java.sql.SQLException ex) {
                        throw new Ili2dbException("failed to delete from " + sqlName, ex);
                    } finally {
                        if (getstmt != null) {
                            try {
                                getstmt.close();
                                getstmt = null;
                            } catch (java.sql.SQLException ex) {
                                EhiLogger.logError(ex);
                            }
                        }
                    }
                }
                try {
                    importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
                } catch (SQLException e) {
                    EhiLogger.logError(e);
                } catch (ConverterException e) {
                    EhiLogger.logError(e);
                }
            }
        } else if (functionCode == Config.FC_UPDATE) {
            if (datasetName == null) {
                throw new Ili2dbException("update requires a dataset name");
            }
            try {
                datasetSqlId = Ili2db.getDatasetId(datasetName, conn, config);
                if (datasetSqlId != null) {
                } else {
                    datasetSqlId = oidPool.newObjSqlId();
                    writeDataset(datasetSqlId, datasetName);
                }
                importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
            } catch (SQLException e) {
                EhiLogger.logError(e);
            } catch (ConverterException e) {
                EhiLogger.logError(e);
            }
        } else if (functionCode == Config.FC_IMPORT) {
            try {
                if (datasetName == null) {
                    datasetSqlId = oidPool.newObjSqlId();
                    if (xtffilename != null) {
                        datasetName = new java.io.File(xtffilename).getName() + "-" + Long.toString(datasetSqlId);
                    } else {
                        datasetName = Long.toString(datasetSqlId);
                    }
                } else {
                    datasetSqlId = Ili2db.getDatasetId(datasetName, conn, config);
                    if (datasetSqlId != null) {
                        throw new Ili2dbException("dataset " + datasetName + " already exists");
                    } else {
                        datasetSqlId = oidPool.newObjSqlId();
                    }
                }
                writeDataset(datasetSqlId, datasetName);
                importSqlId = writeImportStat(datasetSqlId, xtffilename, today, dbusr);
            } catch (SQLException e) {
                EhiLogger.logError(e);
            } catch (ConverterException e) {
                EhiLogger.logError(e);
            }
        } else {
            throw new IllegalArgumentException("unexpected function code " + functionCode);
        }
        if (functionCode == Config.FC_DELETE) {
            return;
        }
        if (reader instanceof ItfReader) {
            ((ItfReader) reader).setBidPrefix(datasetName);
        } else if (reader instanceof ItfReader2) {
            ((ItfReader2) reader).setBidPrefix(datasetName);
        }
        ch.interlis.iox_j.validator.Validator validator = null;
        if (config.isValidation()) {
            ValidationConfig modelConfig = new ValidationConfig();
            modelConfig.mergeIliMetaAttrs(td);
            String configFilename = config.getValidConfigFile();
            if (configFilename != null) {
                try {
                    modelConfig.mergeConfigFile(new File(configFilename));
                } catch (FileNotFoundException e) {
                    EhiLogger.logError("validator config file <" + configFilename + "> not found");
                }
            }
            modelConfig.setConfigValue(ValidationConfig.PARAMETER, ValidationConfig.AREA_OVERLAP_VALIDATION, config.isDisableAreaValidation() ? ValidationConfig.OFF : null);
            modelConfig.setConfigValue(ValidationConfig.PARAMETER, ValidationConfig.DEFAULT_GEOMETRY_TYPE_VALIDATION, config.isSkipGeometryErrors() ? ValidationConfig.OFF : null);
            modelConfig.setConfigValue(ValidationConfig.PARAMETER, ValidationConfig.ALLOW_ONLY_MULTIPLICITY_REDUCTION, config.isOnlyMultiplicityReduction() ? ValidationConfig.ON : null);
            IoxLogging errHandler = new ch.interlis.iox_j.logging.Log2EhiLogger();
            LogEventFactory errFactory = new LogEventFactory();
            errFactory.setDataSource(xtffilename);
            PipelinePool pipelinePool = new PipelinePool();
            validator = new ch.interlis.iox_j.validator.Validator(td, modelConfig, errHandler, errFactory, pipelinePool, config);
            if (reader instanceof ItfReader2) {
                ((ItfReader2) reader).setIoxDataPool(pipelinePool);
            }
        }
        // setup list of objects that have external/forward references
        delayedObjects = new ArrayList<FixIomObjectExtRefs>();
        HashMap<String, ClassStat> objStat = null;
        StartBasketEvent basket = null;
        // more baskets?
        IoxEvent event = reader.read();
        try {
            boolean skipBasket = false;
            while (event != null) {
                if (event instanceof StartBasketEvent) {
                    basket = (StartBasketEvent) event;
                    // do not import this this basket?
                    if ((limitedToBids != null && !limitedToBids.contains(basket.getBid())) || (limitedToTopics != null && !limitedToTopics.contains(basket.getType()))) {
                        // do not import this basket
                        skipBasket = true;
                        EhiLogger.logState("Skip Basket " + basket.getType() + "(oid " + basket.getBid() + ")");
                        if (validator != null)
                            validator.validate(event);
                        if (languageFilter != null) {
                            event = languageFilter.filter(event);
                        }
                    } else {
                        // import this basket
                        EhiLogger.logState("Basket " + basket.getType() + "(oid " + basket.getBid() + ")...");
                        skipBasket = false;
                        try {
                            if (validator != null)
                                validator.validate(event);
                            if (languageFilter != null) {
                                event = languageFilter.filter(event);
                            }
                            Long existingBasketSqlId = null;
                            if (functionCode == Config.FC_UPDATE) {
                                // read existing oid/sqlid mapping (but might also be a new basket)
                                existingObjectsOfCurrentBasket = new HashMap<String, HashSet<Long>>();
                                existingBasketSqlId = readExistingSqlObjIds(reader instanceof ItfReader, basket.getBid());
                                if (existingBasketSqlId == null) {
                                    // new basket
                                    basketSqlId = oidPool.getBasketSqlId(basket.getBid());
                                } else {
                                    // existing basket
                                    basketSqlId = existingBasketSqlId;
                                    // drop existing structeles
                                    dropExistingStructEles(basket.getType(), basketSqlId);
                                }
                            } else {
                                basketSqlId = oidPool.getBasketSqlId(basket.getBid());
                            }
                            if (attachmentKey == null) {
                                if (xtffilename != null) {
                                    attachmentKey = new java.io.File(xtffilename).getName() + "-" + Long.toString(basketSqlId);
                                } else {
                                    attachmentKey = Long.toString(basketSqlId);
                                }
                                config.setAttachmentKey(attachmentKey);
                            }
                            if (existingBasketSqlId == null) {
                                writeBasket(datasetSqlId, basket, basketSqlId, attachmentKey);
                            } else {
                            // TODO update attachmentKey of existing basket
                            }
                        } catch (SQLException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        } catch (ConverterException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        }
                        startTid = oidPool.getLastSqlId();
                        objStat = new HashMap<String, ClassStat>();
                        objCount = 0;
                    }
                } else if (event instanceof EndBasketEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (languageFilter != null) {
                        event = languageFilter.filter(event);
                    }
                    if (reader instanceof ItfReader2) {
                        ArrayList<IoxInvalidDataException> dataerrs = ((ItfReader2) reader).getDataErrs();
                        if (dataerrs.size() > 0) {
                            if (!skipBasket) {
                                if (!config.isSkipGeometryErrors()) {
                                    for (IoxInvalidDataException dataerr : dataerrs) {
                                        EhiLogger.logError(dataerr);
                                    }
                                }
                            }
                            ((ItfReader2) reader).clearDataErrs();
                        }
                    }
                    if (!skipBasket) {
                        // fix external/forward references
                        ArrayList<FixIomObjectExtRefs> fixedObjects = new ArrayList<FixIomObjectExtRefs>();
                        for (FixIomObjectExtRefs fixref : delayedObjects) {
                            boolean skipObj = false;
                            for (IomObject ref : fixref.getRefs()) {
                                String xtfid = ref.getobjectrefoid();
                                Viewable aclass = fixref.getTargetClass(ref);
                                String rootClassName = Ili2cUtility.getRootViewable(aclass).getScopedName(null);
                                if (oidPool.containsXtfid(rootClassName, xtfid)) {
                                // reference now resolvable
                                } else {
                                    // reference not yet known, try to resolve again at end of transfer
                                    skipObj = true;
                                }
                            }
                            if (!skipObj) {
                                doObject(datasetName, basketSqlId, objPool.get(fixref.getRootTid()), objStat);
                                fixedObjects.add(fixref);
                            }
                        }
                        delayedObjects.removeAll(fixedObjects);
                        if (functionCode == Config.FC_UPDATE) {
                            // delete no longer existing objects
                            deleteExisitingObjects(existingObjectsOfCurrentBasket);
                        }
                        // TODO update import counters
                        endTid = oidPool.getLastSqlId();
                        try {
                            String filename = null;
                            if (xtffilename != null) {
                                filename = new java.io.File(xtffilename).getName();
                            }
                            long importId = writeImportBasketStat(importSqlId, basketSqlId, startTid, endTid, objCount);
                            saveObjStat(stat, importId, basket.getBid(), basketSqlId, filename, basket.getType(), objStat);
                        } catch (SQLException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        } catch (ConverterException ex) {
                            EhiLogger.logError("Basket " + basket.getType() + "(oid " + basket.getBid() + ")", ex);
                        }
                    }
                    skipBasket = false;
                } else if (event instanceof ObjectEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (!skipBasket) {
                        if (languageFilter != null) {
                            event = languageFilter.filter(event);
                        }
                        objCount++;
                        IomObject iomObj = ((ObjectEvent) event).getIomObject();
                        if (allReferencesKnown(basketSqlId, iomObj)) {
                            // translate object
                            doObject(datasetName, basketSqlId, iomObj, objStat);
                        }
                    }
                } else if (event instanceof EndTransferEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (languageFilter != null) {
                        event = languageFilter.filter(event);
                    }
                    {
                        for (FixIomObjectExtRefs fixref : delayedObjects) {
                            boolean skipObj = false;
                            for (IomObject ref : fixref.getRefs()) {
                                String xtfid = ref.getobjectrefoid();
                                Viewable aclass = fixref.getTargetClass(ref);
                                String rootClassName = Ili2cUtility.getRootViewable(aclass).getScopedName(null);
                                if (oidPool.containsXtfid(rootClassName, xtfid)) {
                                // skip it; now resolvable
                                } else {
                                    // object in another basket
                                    if (readIliTid || (aclass instanceof AbstractClassDef && ((AbstractClassDef) aclass).getOid() != null)) {
                                        // read object
                                        Long sqlid = readObjectSqlid(aclass, xtfid);
                                        if (sqlid == null) {
                                            if (!ignoreUnresolvedReferences) {
                                                EhiLogger.logError("unknown referenced object " + aclass.getScopedName(null) + " TID " + xtfid + " referenced from " + fixref.getRootTag() + " TID " + fixref.getRootTid());
                                                referrs = true;
                                                skipObj = true;
                                            }
                                        }
                                    } else {
                                        if (!ignoreUnresolvedReferences) {
                                            EhiLogger.logError("unknown referenced object " + aclass.getScopedName(null) + " TID " + xtfid + " referenced from " + fixref.getRootTag() + " TID " + fixref.getRootTid());
                                            referrs = true;
                                            skipObj = true;
                                        }
                                    }
                                }
                            }
                            if (!skipObj) {
                                objStat = stat.get(fixref.getBasketSqlId()).getObjStat();
                                doObject(datasetName, fixref.getBasketSqlId(), objPool.get(fixref.getRootTid()), objStat);
                            }
                        }
                    }
                    break;
                } else if (event instanceof StartTransferEvent) {
                    if (validator != null)
                        validator.validate(event);
                    if (languageFilter != null) {
                        event = languageFilter.filter(event);
                    }
                }
                event = reader.read();
            }
            if (referrs) {
                throw new IoxException("dangling references");
            }
        } finally {
            if (validator != null) {
                validator.close();
                validator = null;
            }
            if (languageFilter != null) {
                languageFilter.close();
            }
        }
    } finally {
        recman.close();
    }
}