Examples with MasterDetailsProcessor net.idea.modbcum.p.MasterDetailsProcessor used on opensource projects

Example 1 with MasterDetailsProcessor

use of net.idea.modbcum.p.MasterDetailsProcessor in project ambit-mirror by ideaconsult.

the class CompoundJSONReporter method configureCollectionProcessors.

@Override
protected void configureCollectionProcessors(String baseURI) {
    if (folders != null && folders.length > 0) {
        CollectionsByChemical collections = new CollectionsByChemical(null);
        collections.setValue(folders);
        MasterDetailsProcessor<IStructureRecord, IFacet<String>, IQueryCondition> facetReader = new MasterDetailsProcessor<IStructureRecord, IFacet<String>, IQueryCondition>(collections) {

            /**
             */
            private static final long serialVersionUID = 3644502232491880656L;

            @Override
            protected IStructureRecord processDetail(IStructureRecord master, IFacet<String> detail) throws Exception {
                master.clearFacets();
                master.addFacet(detail);
                return master;
            }
        };
        facetReader.setCloseConnection(false);
        getProcessors().add(facetReader);
    }
    if (bundles != null && bundles.length > 0) {
        ChemicalRoleByBundle q = new ChemicalRoleByBundle(baseURI);
        q.setValue(bundles[0]);
        MasterDetailsProcessor<IStructureRecord, BundleRoleFacet, IQueryCondition> bundleReader = new MasterDetailsProcessor<IStructureRecord, BundleRoleFacet, IQueryCondition>(q) {

            /**
             */
            private static final long serialVersionUID = -5489861883749717778L;

            @Override
            protected IStructureRecord processDetail(IStructureRecord master, BundleRoleFacet detail) throws Exception {
                master.clearFacets();
                master.addFacet(detail);
                return master;
            }

            @Override
            public IStructureRecord process(IStructureRecord target) throws Exception {
                return super.process(target);
            }
        };
        bundleReader.setCloseConnection(false);
        getProcessors().add(bundleReader);
    }
}

Example 2 with MasterDetailsProcessor

use of net.idea.modbcum.p.MasterDetailsProcessor in project ambit-mirror by ideaconsult.

the class SubstanceDatasetResource method getPropertyProcessors.

protected IProcessor getPropertyProcessors(final boolean removeIdentifiers, final boolean removeStringProperties) {
    IQueryRetrieval<ProtocolEffectRecord<String, String, String>> queryP = getEffectQuery();
    MasterDetailsProcessor<SubstanceRecord, ProtocolEffectRecord<String, String, String>, IQueryCondition> effectReader = new MasterDetailsProcessor<SubstanceRecord, ProtocolEffectRecord<String, String, String>, IQueryCondition>(queryP) {

        /**
         */
        private static final long serialVersionUID = -7354966336095750101L;

        ProtocolEffectRecord2SubstanceProperty processor = new ProtocolEffectRecord2SubstanceProperty();

        @Override
        public SubstanceRecord process(SubstanceRecord target) throws Exception {
            if (target == null || target.getSubstanceUUID() == null)
                return target;
            else
                return super.process(target);
        }

        @Override
        protected SubstanceRecord processDetail(SubstanceRecord master, ProtocolEffectRecord<String, String, String> detail) throws Exception {
            if (master.getIdsubstance() <= 0)
                return master;
            if (detail != null) {
                if (detail.getTextValue() != null && detail.getTextValue().toString().startsWith("{")) {
                    JsonNode node = dx.readTree(new StringReader(detail.getTextValue().toString()));
                    List<String> guideline = detail.getProtocol().getGuideline();
                    ILiteratureEntry ref = LiteratureEntry.getInstance(guideline == null ? null : guideline.size() == 0 ? null : guideline.get(0), guideline == null ? null : guideline.size() == 0 ? null : guideline.get(0));
                    Iterator<Entry<String, JsonNode>> i = node.fields();
                    while (i.hasNext()) {
                        Entry<String, JsonNode> val = i.next();
                        SubstanceProperty key = new SubstanceProperty(detail.getProtocol().getTopCategory(), detail.getProtocol().getCategory(), val.getKey(), detail.getUnit(), ref);
                        try {
                            key.setStudyResultType(_r_flags.valueOf(detail.getStudyResultType().replace(":", "").replace("_", "").replace(" ", "").replace("-", "").replace(")", "").replace("(", "")));
                        } catch (Exception x) {
                            key.setStudyResultType(null);
                        }
                        key.setExtendedURI(true);
                        key.setIdentifier(detail.getSampleID() + "/" + val.getKey());
                        groupProperties.add(key);
                        if (val.getValue().get(EffectRecord._fields.loValue.name()) != null) {
                            Number num = val.getValue().get(EffectRecord._fields.loValue.name()).asDouble();
                            master.setRecordProperty(key, num);
                            key.setClazz(Number.class);
                        } else {
                            master.setRecordProperty(key, val.getValue().textValue());
                            key.setClazz(String.class);
                        }
                    }
                } else {
                    boolean isTextValue = ((detail.getLoValue() == null) && (detail.getUpValue() == null));
                    if (isTextValue && removeStringProperties)
                        return master;
                    /*
						 * JsonNode conditions = detail.getConditions() == null
						 * ? null : dx.readTree(new StringReader(
						 * detail.getConditions()));
						 * 
						 * PropertyAnnotations ann = new PropertyAnnotations();
						 * 
						 * Iterator<Entry<String, JsonNode>> i = conditions ==
						 * null ? null : conditions.getFields();
						 * 
						 * if (i != null) while (i.hasNext()) { Entry<String,
						 * JsonNode> val = i.next(); if (val.getValue()
						 * instanceof NullNode) continue;
						 * 
						 * if (val.getValue().getTextValue() == null) try {
						 * PropertyAnnotation a = new PropertyAnnotation();
						 * String unit =
						 * val.getValue().get(EffectRecord._fields.unit.name())
						 * == null ? null :
						 * val.getValue().get(EffectRecord._fields
						 * .unit.name()).asText(); a.setPredicate(val.getKey());
						 * if (unit == null)
						 * a.setObject(val.getValue().get(EffectRecord
						 * ._fields.loValue.name()) .asText()); else {
						 * a.setObject(String.format("%s %s",
						 * val.getValue().get(
						 * EffectRecord._fields.loValue.name()).asText(),
						 * unit)); } ann.add(a); } catch (Exception x) { } else
						 * { PropertyAnnotation a = new PropertyAnnotation();
						 * a.setPredicate(val.getKey());
						 * a.setObject(val.getValue().getTextValue());
						 * ann.add(a); }
						 * 
						 * }
						 */
                    ProtocolEffectRecord<String, IParams, String> effect = new ProtocolEffectRecord<String, IParams, String>();
                    ProtocolEffectRecord2SubstanceProperty.copyEffectRecordValues(detail, effect);
                    try {
                        JsonNode conditions = detail.getConditions() == null ? null : dx.readTree(new StringReader(detail.getConditions()));
                        if (conditions instanceof ObjectNode) {
                            effect.setConditions(SubstanceStudyParser.parseParams((ObjectNode) conditions));
                        }
                    } catch (Exception x) {
                        logger.log(Level.FINE, x.getMessage());
                    }
                    SubstanceProperty key = processor.process(effect);
                    key.setIdentifier(key.createHashedIdentifier(effect.getConditions()));
                    Object oldValue = master.getRecordProperty(key);
                    groupProperties.add(key);
                    /*
						 * if (isTextValue) { //textvalue if (oldValue == null)
						 * master.setProperty(key, detail.getTextValue()); else
						 * { master.setProperty(key, String.format( "%s, %s",
						 * oldValue instanceof Number ? nf.format((Number)
						 * oldValue) : oldValue .toString(),
						 * detail.getTextValue())); }
						 * key.setClazz(String.class); } else { //numeric
						 */
                    Value value = processValue(detail, isTextValue);
                    if (value != null)
                        ProtocolEffectRecord2SubstanceProperty.addValues(master, key, value, oldValue);
                // }
                }
            }
            return master;
        }
    };
    return effectReader;
}

Example 3 with MasterDetailsProcessor

use of net.idea.modbcum.p.MasterDetailsProcessor in project ambit-mirror by ideaconsult.

the class Context method parseCommandPreprocessing.

public void parseCommandPreprocessing() throws Exception {
    int pagesize = parsePageSizeParam();
    Set<FPTable> preprocessingOption = new TreeSet<FPTable>();
    /*
		 * try { if ((Boolean)options.getParam(":pubchemfp"))
		 * preprocessingOption.add(FPTable.pc1024); } catch (Exception x) {
		 * x.printStackTrace(); }
		 */
    _preprocessingoptions[] po = _preprocessingoptions.values();
    for (_preprocessingoptions p : po) try {
        if ((Boolean) options.getParam(p.toString())) {
            FPTable[] to = p.getOption();
            for (FPTable t : to) preprocessingOption.add(t);
        }
    } catch (Exception x) {
        logger_cli.log(Level.WARNING, x.toString());
    }
    DbReader<IStructureRecord> batch = new DbReader<IStructureRecord>() {

        /**
         */
        private static final long serialVersionUID = 6777121852891369530L;

        @Override
        public void onItemRead(IStructureRecord input, IBatchStatistics stats) {
            super.onItemRead(input, stats);
            if ((stats.getRecords(RECORDS_STATS.RECORDS_READ) % 5000) == 0)
                try {
                    logger_cli.log(Level.INFO, stats.toString());
                    getConnection().commit();
                } catch (Exception x) {
                    logger_cli.log(Level.WARNING, x.getMessage());
                }
        }

        @Override
        public void onError(IStructureRecord input, Object output, IBatchStatistics stats, Exception x) {
            super.onError(input, output, stats, x);
            logger_cli.log(Level.SEVERE, x.getMessage());
        }
    };
    batch.setProcessorChain(new ProcessorsChain<IStructureRecord, IBatchStatistics, IProcessor>());
    /* structure */
    RetrieveStructure queryP = new RetrieveStructure(true);
    queryP.setFieldname(true);
    queryP.setPageSize(1);
    queryP.setPage(0);
    MasterDetailsProcessor<IStructureRecord, IStructureRecord, IQueryCondition> strucReader = new MasterDetailsProcessor<IStructureRecord, IStructureRecord, IQueryCondition>(queryP) {

        /**
         */
        private static final long serialVersionUID = -5350168222668294207L;

        @Override
        protected void configureQuery(IStructureRecord target, IParameterizedQuery<IStructureRecord, IStructureRecord, IQueryCondition> query) throws AmbitException {
            query.setValue(target);
        // super.configureQuery(target, query);
        }

        @Override
        protected IStructureRecord processDetail(IStructureRecord master, IStructureRecord detail) throws Exception {
            master.setContent(detail.getContent());
            master.setFormat(detail.getFormat());
            master.setType(detail.getType());
            return master;
        }
    };
    strucReader.setCloseConnection(false);
    batch.getProcessorChain().add(strucReader);
    // preprocessing itself
    // query
    IQueryRetrieval<IStructureRecord> query = null;
    AbstractUpdate updateQuery = null;
    if (preprocessingOption.isEmpty())
        preprocessingOption.add(FPTable.inchi);
    if (preprocessingOption.contains(FPTable.inchi)) {
        query = new MissingInChIsQuery("UNKNOWN");
        updateQuery = new UpdateChemical();
        batch.getProcessorChain().add(new DefaultAmbitProcessor<IStructureRecord, IStructureRecord>() {

            /**
             */
            private static final long serialVersionUID = -7628269103516836861L;

            protected transient StructureNormalizer normalizer = new StructureNormalizer();

            @Override
            public IStructureRecord process(IStructureRecord record) throws Exception {
                try {
                    normalizer.process(record);
                    return record;
                } catch (Exception x) {
                    record.setType(STRUC_TYPE.NA);
                    return record;
                }
            }
        });
        batch.getProcessorChain().add(new AbstractUpdateProcessor<Object, IChemical>(OP.CREATE, updateQuery) {

            /**
             */
            private static final long serialVersionUID = 9019409150445247686L;

            @Override
            protected IChemical execute(Object group, IQueryUpdate<Object, IChemical> query) throws SQLException, OperationNotSupportedException, AmbitException {
                if (group instanceof IChemical)
                    query.setObject((IChemical) group);
                return super.execute(group, query);
            }
        });
    } else {
        // add generators
        if (preprocessingOption.contains(FPTable.smarts_accelerator)) {
            query = new MissingFingerprintsQuery(FPTable.smarts_accelerator);
            batch.getProcessorChain().add(new SMARTSPropertiesGenerator());
        }
        if (preprocessingOption.contains(FPTable.fp1024)) {
            query = new FingerprintsByStatus(FPTable.fp1024);
            // updateQuery = new CreateFingerprintChemical(FPTable.fp1024);
            batch.getProcessorChain().add(new BitSetGenerator(FPTable.fp1024));
        }
        if (preprocessingOption.contains(FPTable.sk1024)) {
            query = new FingerprintsByStatus(FPTable.sk1024);
            batch.getProcessorChain().add(new BitSetGenerator(FPTable.sk1024));
        }
        // add writers
        if (preprocessingOption.contains(FPTable.smarts_accelerator)) {
            batch.getProcessorChain().add(new SMARTSAcceleratorWriter());
        }
        if (preprocessingOption.contains(FPTable.fp1024)) {
            batch.getProcessorChain().add(new FP1024Writer(FPTable.fp1024));
        }
        if (preprocessingOption.contains(FPTable.sk1024)) {
            batch.getProcessorChain().add(new FP1024Writer(FPTable.sk1024));
        }
        if (preprocessingOption.contains(FPTable.cf1024)) {
            query = new FingerprintsByStatus(FPTable.cf1024);
            batch.getProcessorChain().add(new BitSetGenerator(FPTable.cf1024));
            batch.getProcessorChain().add(new FP1024Writer(FPTable.cf1024));
        }
    }
    batch.setHandlePrescreen(false);
    Connection c = null;
    DBConnectionConfigurable<Context> dbc = null;
    dbc = getConnection(options.getSQLConfig());
    c = dbc.getConnection();
    c.setAutoCommit(false);
    batch.setCloseConnection(true);
    batch.setConnection(c);
    batch.open();
    IBatchStatistics stats = null;
    try {
        query.setPageSize(pagesize);
        logger_cli.info(query.getSQL());
        try {
            disableIndices(batch.getConnection());
        } catch (Exception x) {
            logger_cli.warning(x.getMessage());
        }
        logger_cli.log(Level.INFO, "MSG_INFO_QUERY", pagesize);
        stats = batch.process(query);
    } catch (Exception x) {
        logger_cli.log(Level.WARNING, x.getMessage(), x);
    } finally {
        try {
            batch.getConnection().commit();
        } catch (Exception x) {
            logger_cli.warning(x.getMessage());
        }
        try {
            enableIndices(batch.getConnection());
        } catch (Exception x) {
            logger_cli.warning(x.getMessage());
        }
        try {
            if (batch != null)
                batch.close();
        } catch (Exception x) {
            logger_cli.warning(x.getMessage());
        }
        if (stats != null)
            logger_cli.log(Level.INFO, stats.toString());
    }
}

Example 4 with MasterDetailsProcessor

use of net.idea.modbcum.p.MasterDetailsProcessor in project ambit-mirror by ideaconsult.

the class SubstanceExportResource method getCompositionProcessors.

@Override
protected void getCompositionProcessors(ProcessorsChain chain) {
    final SubstanceEndpointsBundle bundle = null;
    final ReadSubstanceComposition q = new ReadSubstanceComposition();
    MasterDetailsProcessor<SubstanceRecord, CompositionRelation, IQueryCondition> compositionReader = new MasterDetailsProcessor<SubstanceRecord, CompositionRelation, IQueryCondition>(q) {

        /**
         */
        private static final long serialVersionUID = -4012709744454255487L;

        @Override
        public SubstanceRecord process(SubstanceRecord target) throws Exception {
            if (target == null || (target.getIdsubstance() <= 0))
                return target;
            q.setBundle(bundle);
            if (target.getRelatedStructures() != null)
                target.getRelatedStructures().clear();
            return super.process(target);
        }

        protected SubstanceRecord processDetail(SubstanceRecord target, CompositionRelation detail) throws Exception {
            target.addStructureRelation(detail);
            q.setRecord(null);
            return target;
        }
    };
    chain.add(compositionReader);
    final ReadChemIdentifiersByComposition qids = new ReadChemIdentifiersByComposition();
    MasterDetailsProcessor<SubstanceRecord, IStructureRecord, IQueryCondition> idsReader = new MasterDetailsProcessor<SubstanceRecord, IStructureRecord, IQueryCondition>(qids) {

        private static final long serialVersionUID = -3547633994853667140L;

        @Override
        protected SubstanceRecord processDetail(SubstanceRecord target, IStructureRecord detail) throws Exception {
            return qids.processDetail(target, detail);
        }

        @Override
        public SubstanceRecord process(SubstanceRecord target) throws AmbitException {
            try {
                return super.process(target);
            } catch (Exception x) {
                logger.log(Level.FINEST, x.getMessage());
                return target;
            }
        }
    };
    chain.add(idsReader);
    IQueryRetrieval<ExternalIdentifier> queryP = new ReadSubstanceIdentifiers();
    MasterDetailsProcessor<SubstanceRecord, ExternalIdentifier, IQueryCondition> substanceIdentifiers = new MasterDetailsProcessor<SubstanceRecord, ExternalIdentifier, IQueryCondition>(queryP) {

        /**
         */
        private static final long serialVersionUID = 5246468397385927943L;

        @Override
        protected void configureQuery(SubstanceRecord target, IParameterizedQuery<SubstanceRecord, ExternalIdentifier, IQueryCondition> query) throws AmbitException {
            query.setFieldname(target);
        }

        @Override
        protected SubstanceRecord processDetail(SubstanceRecord target, ExternalIdentifier detail) throws Exception {
            if (target.getExternalids() == null)
                target.setExternalids(new ArrayList<ExternalIdentifier>());
            target.getExternalids().add(detail);
            return target;
        }
    };
    substanceIdentifiers.setCloseConnection(false);
    chain.add(substanceIdentifiers);
}

Example 5 with MasterDetailsProcessor

use of net.idea.modbcum.p.MasterDetailsProcessor in project ambit-mirror by ideaconsult.

the class BundleDatasetResource method getCompositionProcessors.

@Override
protected void getCompositionProcessors(ProcessorsChain chain) {
    final ReadSubstanceComposition q = new ReadSubstanceComposition();
    MasterDetailsProcessor<SubstanceRecord, CompositionRelation, IQueryCondition> compositionReader = new MasterDetailsProcessor<SubstanceRecord, CompositionRelation, IQueryCondition>(q) {

        /**
         */
        private static final long serialVersionUID = -4012709744454255487L;

        @Override
        public SubstanceRecord process(SubstanceRecord target) throws Exception {
            if (target == null || (target.getIdsubstance() <= 0))
                return target;
            q.setBundle(bundle);
            if (target.getRelatedStructures() != null)
                target.getRelatedStructures().clear();
            return super.process(target);
        }

        protected SubstanceRecord processDetail(SubstanceRecord target, CompositionRelation detail) throws Exception {
            target.addStructureRelation(detail);
            q.setRecord(null);
            return target;
        }
    };
    chain.add(compositionReader);
    final ReadChemIdentifiersByComposition qids = new ReadChemIdentifiersByComposition();
    MasterDetailsProcessor<SubstanceRecord, IStructureRecord, IQueryCondition> idsReader = new MasterDetailsProcessor<SubstanceRecord, IStructureRecord, IQueryCondition>(qids) {

        private static final long serialVersionUID = -3547633994853667140L;

        @Override
        protected SubstanceRecord processDetail(SubstanceRecord target, IStructureRecord detail) throws Exception {
            return qids.processDetail(target, detail);
        }

        @Override
        public SubstanceRecord process(SubstanceRecord target) throws AmbitException {
            try {
                return super.process(target);
            } catch (Exception x) {
                logger.log(Level.FINE, x.getMessage());
                return target;
            }
        }
    };
    chain.add(idsReader);
    if (mergeDatasets) {
        final ReadChemPropertiesByComposition qprops = new ReadChemPropertiesByComposition();
        MasterDetailsProcessor<SubstanceRecord, IStructureRecord, IQueryCondition> propsReader = new MasterDetailsProcessor<SubstanceRecord, IStructureRecord, IQueryCondition>(qprops) {

            private static final long serialVersionUID = -3547633994853667140L;

            @Override
            protected SubstanceRecord processDetail(SubstanceRecord target, IStructureRecord detail) throws Exception {
                return qprops.processDetail(target, detail);
            }

            @Override
            public SubstanceRecord process(SubstanceRecord target) throws AmbitException {
                try {
                    if ("multi constituent substance".equals(target.getSubstancetype()) || "UVCB".equals(target.getSubstancetype()))
                        return target;
                    else
                        return super.process(target);
                } catch (Exception x) {
                    logger.log(Level.FINE, x.getMessage());
                    return target;
                }
            }
        };
        chain.add(propsReader);
    }
}