Examples with IBatchStatistics net.idea.modbcum.i.batch.IBatchStatistics used on opensource projects

Example 1 with IBatchStatistics

use of net.idea.modbcum.i.batch.IBatchStatistics in project ambit-mirror by ideaconsult.

the class CallableFileImport method importFile.

public TaskResult importFile(File file) throws Exception {
    try {
        // if target dataset is not defined, create new dataset
        final SourceDataset dataset = targetDataset != null ? targetDataset : datasetMeta(file);
        if (targetDataset == null)
            dataset.setId(-1);
        final BatchDBProcessor<String> batch = new BatchDBProcessor<String>() {

            /**
             */
            private static final long serialVersionUID = -7971761364143510120L;

            @Override
            public Iterator<String> getIterator(IInputState target) throws AmbitException {
                try {
                    File file = ((FileInputState) target).getFile();
                    RDFIteratingReader i = getRDFIterator(file, getReporter().getBaseReference().toString());
                    if (i == null) {
                        IIteratingChemObjectReader ni = getNanoCMLIterator(file, getReporter().getBaseReference().toString());
                        if (ni == null)
                            return super.getIterator(target);
                        else
                            return ni;
                    } else {
                        /*
			     * RDFMetaDatasetIterator datasets = null; try {
			     * datasets = new
			     * RDFMetaDatasetIterator(i.getJenaModel());
			     * datasets
			     * .setBaseReference(getReporter().getBaseReference
			     * ()); while (datasets.hasNext()) { SourceDataset d
			     * = datasets.next(); dataset.setId(d.getId());
			     * dataset.setName(d.getName());
			     * dataset.setTitle(d.getTitle());
			     * dataset.setURL(d.getURL()); } } catch (Exception
			     * x) { x.printStackTrace(); } finally { try {
			     * datasets.close();} catch (Exception x) {} }
			     */
                        return i;
                    }
                } catch (AmbitException x) {
                    throw x;
                } catch (Exception x) {
                    throw new AmbitException(x);
                }
            }

            @Override
            public void onItemProcessed(String input, Object output, IBatchStatistics stats) {
                super.onItemProcessed(input, output, stats);
                if (firstCompoundOnly && (stats.getRecords(RECORDS_STATS.RECORDS_PROCESSED) >= 1)) {
                    cancelled = true;
                    if (output != null)
                        if ((output instanceof ArrayList) && ((ArrayList) output).size() > 0) {
                            if (((ArrayList) output).get(0) instanceof IStructureRecord)
                                recordImported = (IStructureRecord) ((ArrayList) output).get(0);
                        } else if (output instanceof IStructureRecord)
                            recordImported = (IStructureRecord) output;
                }
            }
        };
        batch.setReference(dataset.getReference());
        batch.setConnection(connection);
        final RepositoryWriter writer = new RepositoryWriter();
        writer.setUseExistingStructure(isPropertyOnly());
        writer.setPropertyKey(getMatcher());
        writer.setDataset(dataset);
        final ProcessorsChain<String, IBatchStatistics, IProcessor> chain = new ProcessorsChain<String, IBatchStatistics, IProcessor>();
        chain.add(writer);
        batch.setProcessorChain(chain);
        writer.setConnection(connection);
        FileInputState fin = new FileInputState(file);
        IBatchStatistics stats = batch.process(fin);
        if (firstCompoundOnly) {
            if (recordImported == null)
                throw new Exception("No compound imported");
            if (compoundReporter == null)
                compoundReporter = new ConformerURIReporter("", null, false);
            try {
                batch.close();
            } catch (Exception xx) {
            }
            return new TaskResult(compoundReporter.getURI(recordImported));
        } else {
            ReadDataset q = new ReadDataset();
            q.setValue(dataset);
            QueryExecutor<ReadDataset> x = new QueryExecutor<ReadDataset>();
            x.setConnection(connection);
            ResultSet rs = x.process(q);
            ISourceDataset newDataset = null;
            while (rs.next()) {
                newDataset = q.getObject(rs);
                break;
            }
            x.closeResults(rs);
            x.setConnection(null);
            if (newDataset == null)
                throw new ResourceException(Status.SUCCESS_NO_CONTENT);
            if (reporter == null)
                reporter = new DatasetURIReporter<IQueryRetrieval<ISourceDataset>, ISourceDataset>();
            try {
                batch.close();
            } catch (Exception xx) {
            }
            return new TaskResult(reporter.getURI(newDataset));
        }
    } catch (ResourceException x) {
        throw x;
    } catch (Exception x) {
        throw new ResourceException(new Status(Status.SERVER_ERROR_INTERNAL, x.getMessage()));
    } finally {
        try {
            connection.close();
        } catch (Exception x) {
        }
        connection = null;
    }
}

Example 2 with IBatchStatistics

use of net.idea.modbcum.i.batch.IBatchStatistics in project ambit-mirror by ideaconsult.

the class CallableFingerprintsModelCreator method createProcessors.

protected ProcessorsChain<IStructureRecord, IBatchStatistics, IProcessor> createProcessors() throws Exception {
    ProcessorsChain<IStructureRecord, IBatchStatistics, IProcessor> p1 = new ProcessorsChain<IStructureRecord, IBatchStatistics, IProcessor>();
    p1.add(new ProcessorStructureRetrieval());
    p1.add(new MoleculeReader());
    p1.add(new FingerprintGenerator(new Fingerprinter()));
    p1.add(new DefaultAmbitProcessor<BitSet, BitSet>() {

        /**
         */
        private static final long serialVersionUID = -2978697147490545478L;

        public BitSet process(BitSet target) throws AmbitException {
            builder.getTrainingData().add(target);
            return target;
        }
    });
    p1.setAbortOnError(true);
    return p1;
}

Example 3 with IBatchStatistics

use of net.idea.modbcum.i.batch.IBatchStatistics in project ambit-mirror by ideaconsult.

the class CallableUpdateDataset method createProcessors.

@Override
protected ProcessorsChain<String, IBatchStatistics, IProcessor> createProcessors() throws Exception {
    final RepositoryWriter writer = new RepositoryWriter();
    writer.setDataset(dataset);
    final ProcessorsChain<String, IBatchStatistics, IProcessor> chain = new ProcessorsChain<String, IBatchStatistics, IProcessor>();
    chain.add(writer);
    return chain;
}

Example 4 with IBatchStatistics

use of net.idea.modbcum.i.batch.IBatchStatistics in project ambit-mirror by ideaconsult.

the class BatchProcessor method process.

/*
	protected Iterator getIterator(IInputState target) throws AmbitException {
		return target.getReader(); 
	}
	
	protected void closeIterator(Iterator iterator) throws AmbitException {
		iterator.close();
	}
	*/
public IBatchStatistics process(INPUT target) throws AmbitException {
    try {
        DefaultBatchStatistics stats = new DefaultBatchStatistics();
        stats.setResultCaption("Read");
        stats.setFrequency(1);
        Iterator reader = getIterator(target);
        ProcessorsChain<Target, IBatchStatistics, IProcessor> processor = getProcessorChain();
        if (processor == null)
            throw new AmbitException("Processor not defined");
        long started = System.currentTimeMillis();
        while (reader.hasNext() && !cancelled) {
            if ((stats.getRecords(IBatchStatistics.RECORDS_STATS.RECORDS_READ) % stats.getFrequency()) == 0)
                propertyChangeSupport.firePropertyChange(PROPERTY_BATCHSTATS, null, stats);
            Object object = null;
            long ms = System.currentTimeMillis();
            try {
                object = reader.next();
                stats.increment(IBatchStatistics.RECORDS_STATS.RECORDS_READ);
                stats.incrementTimeElapsed(IBatchStatistics.RECORDS_STATS.RECORDS_READ, System.currentTimeMillis() - ms);
            } catch (Exception x) {
                stats.increment(IBatchStatistics.RECORDS_STATS.RECORDS_ERROR);
                stats.incrementTimeElapsed(IBatchStatistics.RECORDS_STATS.RECORDS_ERROR, System.currentTimeMillis() - ms);
                continue;
            }
            ms = System.currentTimeMillis();
            try {
                processor.process((Target) object);
                stats.increment(IBatchStatistics.RECORDS_STATS.RECORDS_PROCESSED);
                stats.incrementTimeElapsed(IBatchStatistics.RECORDS_STATS.RECORDS_PROCESSED, System.currentTimeMillis() - ms);
            } catch (Exception x) {
                stats.increment(IBatchStatistics.RECORDS_STATS.RECORDS_ERROR);
                stats.incrementTimeElapsed(IBatchStatistics.RECORDS_STATS.RECORDS_ERROR, System.currentTimeMillis() - ms);
                continue;
            }
            long elapsed = System.currentTimeMillis() - started;
            if ((timeout > 0) && (elapsed > timeout)) {
                stats.increment(IBatchStatistics.RECORDS_STATS.RECORDS_ERROR);
                stats.incrementTimeElapsed(IBatchStatistics.RECORDS_STATS.RECORDS_ERROR, System.currentTimeMillis() - ms);
                break;
            }
        }
        propertyChangeSupport.firePropertyChange(PROPERTY_BATCHSTATS, null, stats);
        closeIterator(reader);
        return stats;
    } catch (Exception x) {
        throw new AmbitException(x);
    }
}

Example 5 with IBatchStatistics

use of net.idea.modbcum.i.batch.IBatchStatistics in project ambit-mirror by ideaconsult.

the class DbReaderTest method testProcess.

@Test
public void testProcess() throws Exception {
    setUpDatabaseFromResource("ambit2/db/processors/test/descriptors-datasets.xml");
    IDatabaseConnection c = getConnection();
    ITable names = c.createQueryTable("EXPECTED_NAMES", "SELECT * FROM properties");
    Assert.assertEquals(5, names.getRowCount());
    ITable values = c.createQueryTable("EXPECTED_VALUES", "SELECT * FROM property_values");
    Assert.assertEquals(2, values.getRowCount());
    ProcessorsChain<IStructureRecord, IBatchStatistics, IProcessor> p = new ProcessorsChain<IStructureRecord, IBatchStatistics, IProcessor>();
    p.add(new ProcessorStructureRetrieval());
    p.add(new IProcessor<IStructureRecord, IStructureRecord>() {

        public boolean isEnabled() {
            return true;
        }

        public IStructureRecord process(IStructureRecord target) throws AmbitException {
            // System.out.println(target.getContent());
            return target;
        }

        public void setEnabled(boolean value) {
        }

        public long getID() {
            return 0;
        }

        @Override
        public void open() throws Exception {
        }

        @Override
        public void close() throws Exception {
        }
    });
    p.add(new MoleculeReader());
    p.add(new AtomConfigurator());
    p.add(new FunctionalGroup("Test", "P", "Test"));
    p.add(new IProcessor<VerboseDescriptorResult, String>() {

        /**
         */
        private static final long serialVersionUID = -3923864774580483348L;

        public String process(VerboseDescriptorResult target) throws AmbitException {
            Assert.assertEquals("1", target.getResult().toString());
            return target.toString();
        }

        public boolean isEnabled() {
            return true;
        }

        public void setEnabled(boolean value) {
        }

        public long getID() {
            return 0;
        }

        @Override
        public void open() throws Exception {
        }

        @Override
        public void close() throws Exception {
        }
    });
    batch.setProcessorChain(p);
    QueryStructureByID query = new QueryStructureByID(100215);
    batch.setConnection(c.getConnection());
    batch.open();
    IBatchStatistics stats = batch.process(query);
    Assert.assertEquals(1, stats.getRecords(IBatchStatistics.RECORDS_STATS.RECORDS_READ));
    Assert.assertEquals(1, stats.getRecords(IBatchStatistics.RECORDS_STATS.RECORDS_PROCESSED));
// C20H20BrP
}