Examples with ObservationResponse ca.nrc.cadc.caom2.ObservationResponse used on opensource projects

Example 1 with ObservationResponse

use of ca.nrc.cadc.caom2.ObservationResponse in project caom2db by opencadc.

the class AbstractObservationDAOTest method testPutGetDelete.

@Test
public void testPutGetDelete() {
    try {
        Observation orig = getTestObservation(false, 5, false, true);
        // !EXISTS
        Assert.assertNull(dao.getState(orig.getURI()));
        log.info("not-exists");
        ObservationResponse notFound = dao.getObservationResponse(orig.getURI());
        Assert.assertNotNull("wrapper", notFound);
        Assert.assertNotNull("wrapper", notFound.observationState);
        Assert.assertNull("wrapped", notFound.observation);
        // PUT
        dao.put(orig);
        // this is so we can detect incorrect timestamp round trips
        // caused by assigning something other than what was stored
        Thread.sleep(2 * TIME_TOLERANCE);
        log.info("put + sleep");
        // EXISTS
        Assert.assertNotNull("exists-by-uri", dao.getState(orig.getURI()));
        Assert.assertNotNull("exists-by-uuid", dao.getState(orig.getID()));
        log.info("exists");
        // GET by URI
        ObservationState st = dao.getState(orig.getURI());
        log.info("found: " + st);
        Assert.assertNotNull("found ObservationState by URI", st);
        ObservationResponse rs = dao.getObservationResponse(st);
        log.info("found: " + rs);
        Assert.assertNotNull("found ObservationResponse by URI", rs);
        Observation retrieved = rs.observation;
        log.info("found: " + retrieved);
        Assert.assertNotNull("found Observation by URI", retrieved);
        log.info("retrieved by URI");
        testEqual(orig, retrieved);
        log.info("equal");
        // GET by ID
        retrieved = dao.get(orig.getID());
        Assert.assertNotNull("found by ID", retrieved);
        log.info("retrieved by UUID");
        testEqual(orig, retrieved);
        log.info("equal");
        // DELETE by ID
        dao.delete(orig.getID());
        // EXISTS
        Assert.assertNull(dao.getState(orig.getURI()));
        log.info("delete & not-exists");
        log.info("check deletion track: " + orig.getID());
        DeletedEntity de = ded.get(DeletedObservation.class, orig.getID());
        Assert.assertNotNull("deletion tracker", de);
        Assert.assertEquals("deleted.id", orig.getID(), de.getID());
        Assert.assertNotNull("deleted.lastModified", de.lastModified);
        DeletedObservation doe = (DeletedObservation) de;
        Assert.assertEquals("deleted.uri", orig.getURI(), doe.getURI());
        Assert.assertFalse("open transaction", txnManager.isOpen());
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    }
}

Example 2 with ObservationResponse

use of ca.nrc.cadc.caom2.ObservationResponse in project caom2db by opencadc.

the class AbstractObservationDAOTest method testGetObservationList.

@Test
public void testGetObservationList() {
    try {
        log.info("testGetObservationList");
        Integer batchSize = new Integer(3);
        String collection = "FOO";
        Observation o1 = new SimpleObservation(collection, "obs1");
        Observation o2 = new SimpleObservation(collection, "obsA");
        Observation o3 = new SimpleObservation(collection, "obs2");
        Observation o4 = new SimpleObservation(collection, "obsB");
        Observation o5 = new SimpleObservation(collection, "obs3");
        // txnManager.startTransaction();
        dao.put(o1);
        Thread.sleep(10L);
        dao.put(o2);
        Thread.sleep(10L);
        dao.put(o3);
        Thread.sleep(10L);
        dao.put(o4);
        Thread.sleep(10L);
        dao.put(o5);
        // txnManager.commitTransaction();
        List<ObservationResponse> obs;
        // get first batch
        obs = dao.getList(collection, null, null, batchSize);
        Assert.assertNotNull(obs);
        Assert.assertEquals(3, obs.size());
        Assert.assertEquals(o1.getURI(), obs.get(0).observation.getURI());
        Assert.assertEquals(o2.getURI(), obs.get(1).observation.getURI());
        Assert.assertEquals(o3.getURI(), obs.get(2).observation.getURI());
        // get next batch
        obs = dao.getList(collection, o3.getMaxLastModified(), null, batchSize);
        Assert.assertNotNull(obs);
        // o3 gets picked up by the >=
        Assert.assertEquals(3, obs.size());
        Assert.assertEquals(o3.getURI(), obs.get(0).observation.getURI());
        Assert.assertEquals(o4.getURI(), obs.get(1).observation.getURI());
        Assert.assertEquals(o5.getURI(), obs.get(2).observation.getURI());
        // txnManager.startTransaction();
        dao.delete(o1.getURI());
        dao.delete(o2.getURI());
        dao.delete(o3.getURI());
        dao.delete(o4.getURI());
        dao.delete(o5.getURI());
    // txnManager.commitTransaction();
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        if (txnManager.isOpen())
            try {
                txnManager.rollbackTransaction();
            } catch (Throwable t) {
                log.error("failed to rollback transaction", t);
            }
        Assert.fail("unexpected exception: " + unexpected);
    }
}

Example 3 with ObservationResponse

use of ca.nrc.cadc.caom2.ObservationResponse in project caom2db by opencadc.

the class ArtifactValidator method getLogicalMetadata.

private TreeSet<ArtifactMetadata> getLogicalMetadata() throws Exception {
    TreeSet<ArtifactMetadata> result = new TreeSet<>(ArtifactMetadata.getComparator());
    if (StringUtil.hasText(source)) {
        // use database <server.database.schema>
        // HarvestSkipURI table is not supported in 'diff' mode, i.e. reportOnly = true
        this.supportSkipURITable = !reportOnly;
        long t1 = System.currentTimeMillis();
        List<ObservationState> states = observationDAO.getObservationList(collection, null, null, null);
        long t2 = System.currentTimeMillis();
        long dt = t2 - t1;
        log.info("get-state-list: size=" + states.size() + " in " + dt + " ms");
        int depth = 3;
        ListIterator<ObservationState> iter = states.listIterator();
        t1 = System.currentTimeMillis();
        while (iter.hasNext()) {
            ObservationState s = iter.next();
            // GC
            iter.remove();
            ObservationResponse resp = observationDAO.getObservationResponse(s, depth);
            if (resp == null) {
                log.error("Null response from Observation DAO, ObservationURI: " + s.getURI().toString() + ", depth: " + depth);
            } else if (resp.observation == null) {
                log.error("Observation is null, ObservationURI: " + s.getURI().toString() + ", depth: " + depth);
            } else {
                for (Plane plane : resp.observation.getPlanes()) {
                    for (Artifact artifact : plane.getArtifacts()) {
                        String observationID = s.getURI().getObservationID();
                        result.add(getMetadata(observationID, artifact, plane.dataRelease, plane.metaRelease));
                    }
                }
            }
        }
        log.info("Finished logical metadata query in " + (System.currentTimeMillis() - t1) + " ms");
    } else {
        this.supportSkipURITable = false;
        if (caomTapResourceID != null) {
            // source is a TAP resource ID
            AuthMethod authMethod = AuthenticationUtil.getAuthMethodFromCredentials(AuthenticationUtil.getCurrentSubject());
            TapClient tapClient = new TapClient(caomTapResourceID);
            try {
                this.caomTapURL = tapClient.getSyncURL(authMethod);
            } catch (ResourceNotFoundException ex) {
                if (ex.getMessage().contains("with password")) {
                    throw new ResourceNotFoundException("TAP service for " + caomTapResourceID + " does not support password authentication.", ex);
                }
            }
        }
        // source is a TAP service URL or a TAP resource ID
        String adql = "select distinct(a.uri), a.contentChecksum, a.contentLength, a.contentType, o.observationID, " + "a.productType, a.releaseType, p.dataRelease, p.metaRelease " + "from caom2.Artifact a " + "join caom2.Plane p on a.planeID = p.planeID " + "join caom2.Observation o on p.obsID = o.obsID " + "where o.collection='" + collection + "'";
        log.debug("logical query: " + adql);
        long start = System.currentTimeMillis();
        result = query(caomTapURL, adql);
        log.info("Finished caom2 query in " + (System.currentTimeMillis() - start) + " ms");
    }
    return result;
}

Example 4 with ObservationResponse

use of ca.nrc.cadc.caom2.ObservationResponse in project caom2db by opencadc.

the class ObservationValidator method doit.

private Aggregate doit() {
    Aggregate ret = new Aggregate();
    if (!ready) {
        log.error("Observation Validator not ready");
        ret.abort = true;
        return ret;
    }
    long t = System.currentTimeMillis();
    long timeQuery = -1;
    long timeValidation = -1;
    int expectedNum = Integer.MAX_VALUE;
    if (batchSize != null) {
        expectedNum = batchSize.intValue();
    }
    log.debug("expectedNum: " + expectedNum);
    try {
        // hint
        System.gc();
        t = System.currentTimeMillis();
        progressRecord = readProgressFile();
        startDate = progressRecord.curLastModified;
        if (firstIteration) {
            if (full) {
                startDate = null;
            } else if (this.minDate != null) {
                startDate = this.minDate;
            }
            endDate = this.maxDate;
        }
        firstIteration = false;
        log.info("validation window: " + format(startDate) + " :: " + format(endDate) + " [" + batchSize + "]");
        List<ObservationResponse> obsList;
        if (srcObservationDAO != null) {
            obsList = srcObservationDAO.getList(src.getCollection(), startDate, endDate, batchSize);
        } else {
            obsList = srcObservationService.getList(src.getCollection(), startDate, endDate, batchSize);
        }
        // avoid re-processing the last successful one stored in progressRecord (normal case because query: >= startDate)
        if (!obsList.isEmpty()) {
            ListIterator<ObservationResponse> iter = obsList.listIterator();
            Observation curBatchLeader = iter.next().observation;
            if (curBatchLeader != null) {
                log.debug("currentBatch: " + curBatchLeader.getURI() + " " + format(curBatchLeader.getMaxLastModified()));
                log.debug("currentBatch: " + curBatchLeader.getURI() + " <need a date here for format asking for URI?>");
                log.debug("progressRecord: " + format(progressRecord.curID) + " " + format(progressRecord.curLastModified));
                if (curBatchLeader.getID().equals(progressRecord.curID) && curBatchLeader.getMaxLastModified().equals(progressRecord.curLastModified)) {
                    iter.remove();
                    expectedNum--;
                }
            }
        }
        ret.found = obsList.size();
        log.debug("observation count: " + obsList.size());
        timeQuery = System.currentTimeMillis() - t;
        t = System.currentTimeMillis();
        ListIterator<ObservationResponse> iter1 = obsList.listIterator();
        while (iter1.hasNext()) {
            boolean clean = true;
            log.debug("next iteration...");
            ObservationResponse ow = iter1.next();
            ObservationURI observationURI = ow.observationState.getURI();
            Observation o = ow.observation;
            // allow garbage collection during loop
            iter1.remove();
            if (o != null) {
                progressRecord.curLastModified = o.getMaxLastModified();
                progressRecord.curID = o.getID();
                log.debug("max last modified: " + o.getMaxLastModified());
                // core validator
                try {
                    log.debug("core validation");
                    CaomValidator.validate(o);
                } catch (IllegalArgumentException coreOops) {
                    clean = false;
                    String str = coreOops.toString();
                    log.error(CORE_ERROR + " invalid observation: " + observationURI + " " + format(o.getMaxLastModified()) + " - " + coreOops.getMessage());
                    if (coreOops.getCause() != null) {
                        log.error("cause: " + coreOops.getCause());
                    }
                    ret.coreErr++;
                }
                // WCS validation
                try {
                    log.debug("wcs validation");
                    for (Plane p : o.getPlanes()) {
                        for (Artifact a : p.getArtifacts()) {
                            CaomWCSValidator.validate(a);
                        }
                    }
                } catch (IllegalArgumentException wcsOops) {
                    clean = false;
                    log.error(CORE_ERROR + " invalid observation: " + observationURI + " " + format(o.getMaxLastModified()) + " - " + wcsOops.getMessage());
                    if (wcsOops.getCause() != null) {
                        log.error("cause: " + wcsOops.getCause());
                    }
                    ret.wcsErr++;
                }
                try {
                    log.debug("checksum validation");
                    validateChecksum(o);
                } catch (MismatchedChecksumException checksumOops) {
                    clean = false;
                    log.error(CHECKSUM_ERROR + " mismatching checksums: " + checksumOops.getMessage() + " " + format(o.getMaxLastModified()));
                    ret.checksumErr++;
                }
                // default is false
                if (computePlaneMetadata) {
                    try {
                        log.debug("computePlaneMetadata: " + o.getURI());
                        for (Plane p : o.getPlanes()) {
                            ComputeUtil.computeTransientState(o, p);
                        }
                    } catch (IllegalArgumentException otherOoops) {
                        clean = false;
                        log.error("COMPUTE - " + observationURI + " " + format(o.getMaxLastModified()) + " - " + otherOoops.getMessage());
                        ret.computeErr++;
                    }
                }
            } else if (ow.error != null) {
                clean = false;
                // unwrap intervening RuntimeException(s)
                Throwable oops = ow.error.getCause();
                while (oops.getCause() != null && oops instanceof RuntimeException) {
                    oops = oops.getCause();
                }
                log.error("TRANSIENT - failed to read observation: " + observationURI + " - " + oops.getMessage());
                ret.srcErr++;
            }
            writeProgressFile(progressRecord);
            if (clean == true) {
                ret.passed++;
            } else {
                ret.failed++;
            }
            timeValidation += System.currentTimeMillis() - t;
        }
        if (ret.found < expectedNum) {
            ret.done = true;
        }
    } catch (InterruptedException | ExecutionException e) {
        log.error("SEVERE PROBLEM - ThreadPool harvesting Observations failed: " + e.getMessage());
        ret.abort = true;
        ret.runtime++;
    } catch (IOException | ParseException iope) {
        log.error("SEVERE PROBLEM - error reading from progress file" + iope.toString());
        ret.runtime++;
    } catch (URISyntaxException ue) {
        log.error("SEVERE PROBLEM - error reading observation URI from progress file: " + ue.toString());
        ret.runtime++;
    } finally {
        timeValidation = System.currentTimeMillis() - t;
        ret.processTime = timeQuery + timeValidation;
        log.info("batch stats: " + ret.toString() + " query time: " + timeQuery + "ms validation time: " + timeValidation + "ms\n");
    }
    return ret;
}

Example 5 with ObservationResponse

use of ca.nrc.cadc.caom2.ObservationResponse in project caom2db by opencadc.

the class RepoClient method getList.

public List<ObservationResponse> getList(String collection, Date startDate, Date end, Integer numberOfObservations) throws InterruptedException, ExecutionException {
    // startDate = null;
    // end = df.parse("2017-06-20T09:03:15.360");
    List<ObservationResponse> list = new ArrayList<>();
    List<ObservationState> stateList = getObservationList(collection, startDate, end, numberOfObservations);
    // Create tasks for each file
    List<Callable<ObservationResponse>> tasks = new ArrayList<>();
    // the current subject usually gets propagated into a thread pool, but
    // gets attached
    // when the thread is created so we explicitly pass it it and do another
    // Subject.doAs in
    // case
    // thread pool management is changed
    Subject subjectForWorkerThread = AuthenticationUtil.getCurrentSubject();
    for (ObservationState os : stateList) {
        tasks.add(new Worker(os, subjectForWorkerThread, baseServiceURL.toExternalForm()));
    }
    ExecutorService taskExecutor = null;
    try {
        // Run tasks in a fixed thread pool
        taskExecutor = Executors.newFixedThreadPool(nthreads);
        List<Future<ObservationResponse>> futures;
        futures = taskExecutor.invokeAll(tasks);
        for (Future<ObservationResponse> f : futures) {
            ObservationResponse res = null;
            res = f.get();
            if (f.isDone()) {
                list.add(res);
            }
        }
    } catch (InterruptedException | ExecutionException e) {
        log.error("Error when executing thread in ThreadPool: " + e.getMessage() + " caused by: " + e.getCause().toString());
        throw e;
    } finally {
        if (taskExecutor != null) {
            taskExecutor.shutdown();
        }
    }
    return list;
}