Examples with ObservationState ca.nrc.cadc.caom2.ObservationState used on opensource projects

Example 1 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class ObservationDAO method getStateImpl.

private ObservationState getStateImpl(ObservationURI uri, UUID id) {
    checkInit();
    if (uri == null && id == null) {
        throw new IllegalArgumentException("args cannot be null");
    }
    log.debug("GET: " + uri + " | " + id);
    long t = System.currentTimeMillis();
    try {
        String sql = null;
        if (uri != null) {
            sql = gen.getSelectSQL(uri, 1, false);
        } else {
            sql = gen.getSelectSQL(id, 1, false);
        }
        log.debug("GET: " + sql);
        JdbcTemplate jdbc = new JdbcTemplate(dataSource);
        // ObservationSkeleton skel = (ObservationSkeleton) jdbc.query(sql, new ObservationSkeletonExtractor());
        Observation obs = (Observation) jdbc.query(sql, gen.getObservationExtractor());
        if (obs != null) {
            ObservationState ret = new ObservationState(obs.getURI());
            ret.id = obs.getID();
            ret.accMetaChecksum = obs.getAccMetaChecksum();
            ret.maxLastModified = obs.getMaxLastModified();
            // ret.maxLastModified = skel.maxLastModified;
            return ret;
        }
        return null;
    } finally {
        long dt = System.currentTimeMillis() - t;
        log.debug("GET: " + uri + " | " + id + " " + dt + "ms");
    }
}

Example 2 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class AbstractObservationDAOTest method testPutGetDelete.

@Test
public void testPutGetDelete() {
    try {
        Observation orig = getTestObservation(false, 5, false, true);
        // !EXISTS
        Assert.assertNull(dao.getState(orig.getURI()));
        log.info("not-exists");
        ObservationResponse notFound = dao.getObservationResponse(orig.getURI());
        Assert.assertNotNull("wrapper", notFound);
        Assert.assertNotNull("wrapper", notFound.observationState);
        Assert.assertNull("wrapped", notFound.observation);
        // PUT
        dao.put(orig);
        // this is so we can detect incorrect timestamp round trips
        // caused by assigning something other than what was stored
        Thread.sleep(2 * TIME_TOLERANCE);
        log.info("put + sleep");
        // EXISTS
        Assert.assertNotNull("exists-by-uri", dao.getState(orig.getURI()));
        Assert.assertNotNull("exists-by-uuid", dao.getState(orig.getID()));
        log.info("exists");
        // GET by URI
        ObservationState st = dao.getState(orig.getURI());
        log.info("found: " + st);
        Assert.assertNotNull("found ObservationState by URI", st);
        ObservationResponse rs = dao.getObservationResponse(st);
        log.info("found: " + rs);
        Assert.assertNotNull("found ObservationResponse by URI", rs);
        Observation retrieved = rs.observation;
        log.info("found: " + retrieved);
        Assert.assertNotNull("found Observation by URI", retrieved);
        log.info("retrieved by URI");
        testEqual(orig, retrieved);
        log.info("equal");
        // GET by ID
        retrieved = dao.get(orig.getID());
        Assert.assertNotNull("found by ID", retrieved);
        log.info("retrieved by UUID");
        testEqual(orig, retrieved);
        log.info("equal");
        // DELETE by ID
        dao.delete(orig.getID());
        // EXISTS
        Assert.assertNull(dao.getState(orig.getURI()));
        log.info("delete & not-exists");
        log.info("check deletion track: " + orig.getID());
        DeletedEntity de = ded.get(DeletedObservation.class, orig.getID());
        Assert.assertNotNull("deletion tracker", de);
        Assert.assertEquals("deleted.id", orig.getID(), de.getID());
        Assert.assertNotNull("deleted.lastModified", de.lastModified);
        DeletedObservation doe = (DeletedObservation) de;
        Assert.assertEquals("deleted.uri", orig.getURI(), doe.getURI());
        Assert.assertFalse("open transaction", txnManager.isOpen());
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    }
}

Example 3 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class ArtifactHarvester method runIt.

private Boolean runIt() throws Exception {
    this.downloadCount = 0;
    this.processedCount = 0;
    this.start = new Date();
    try {
        // Determine the state of the last run
        HarvestState state = harvestStateDAO.get(source, STATE_CLASS);
        this.startDate = state.curLastModified;
        // harvest up to a little in the past because the head of
        // the sequence may be volatile
        long fiveMinAgo = System.currentTimeMillis() - 5 * 60000L;
        Date stopDate = new Date(fiveMinAgo);
        if (startDate == null) {
            log.info("harvest window: null " + this.df.format(stopDate) + " [" + this.batchSize + "]");
        } else {
            log.info("harvest window: " + this.df.format(startDate) + " " + this.df.format(stopDate) + " [" + this.batchSize + "]");
        }
        List<ObservationState> observationStates = this.observationDAO.getObservationList(this.collection, this.startDate, stopDate, this.batchSize + 1);
        // HarvestState (normal case because query: >= startDate)
        if (!observationStates.isEmpty()) {
            ListIterator<ObservationState> iter = observationStates.listIterator();
            ObservationState curBatchLeader = iter.next();
            if (curBatchLeader != null) {
                if (state.curLastModified != null) {
                    log.debug("harvesState: " + format(state.curID) + ", " + this.df.format(state.curLastModified));
                }
                if (curBatchLeader.getMaxLastModified().equals(state.curLastModified)) {
                    Observation observation = this.observationDAO.get(curBatchLeader.getID());
                    log.debug("current batch: " + format(observation.getID()) + ", " + this.df.format(curBatchLeader.getMaxLastModified()));
                    if (state.curID != null && state.curID.equals(observation.getID())) {
                        iter.remove();
                    }
                }
            }
        }
        log.info("Found: " + observationStates.size());
        for (ObservationState observationState : observationStates) {
            try {
                this.observationDAO.getTransactionManager().startTransaction();
                Observation observation = this.observationDAO.get(observationState.getID());
                if (observation == null) {
                    log.debug("Observation no longer exists: " + observationState.getURI());
                } else {
                    // will make progress even on failures
                    state.curLastModified = observation.getMaxLastModified();
                    state.curID = observation.getID();
                    for (Plane plane : observation.getPlanes()) {
                        for (Artifact artifact : plane.getArtifacts()) {
                            Date releaseDate = AccessUtil.getReleaseDate(artifact, plane.metaRelease, plane.dataRelease);
                            if (releaseDate == null) {
                                // null date means private
                                log.debug("null release date, skipping");
                            } else {
                                logStart(format(state.curID), artifact);
                                boolean success = true;
                                boolean added = false;
                                String message = null;
                                this.caomChecksum = getMD5Sum(artifact.contentChecksum);
                                if (this.caomChecksum == null) {
                                    this.caomChecksum = "null";
                                }
                                if (artifact.contentLength == null) {
                                    this.caomContentLength = null;
                                } else {
                                    this.caomContentLength = artifact.contentLength;
                                }
                                this.storageContentLength = 0;
                                this.reason = "None";
                                this.errorMessage = null;
                                this.processedCount++;
                                if (releaseDate.after(start)) {
                                    // private and release date is not null, download in the future
                                    this.errorMessage = ArtifactHarvester.PROPRIETARY;
                                }
                                try {
                                    // by default, do not add to the skip table
                                    boolean correctCopy = true;
                                    // artifact is not in storage if storage policy is 'PUBLIC ONLY' and the artifact is proprietary
                                    if ((StoragePolicy.ALL == storagePolicy) || this.errorMessage == null) {
                                        // correctCopy is false if: checksum mismatch, content length mismatch or not in storage
                                        // "not in storage" includes failing to resolve the artifact URI
                                        correctCopy = checkArtifactInStorage(artifact.getURI());
                                        log.debug("Artifact " + artifact.getURI() + " with MD5 " + artifact.contentChecksum + " correct copy: " + correctCopy);
                                    }
                                    if ((StoragePolicy.PUBLIC_ONLY == storagePolicy && ArtifactHarvester.PROPRIETARY.equals(this.errorMessage)) || !correctCopy) {
                                        HarvestSkipURI skip = harvestSkipURIDAO.get(source, STATE_CLASS, artifact.getURI());
                                        if (skip == null) {
                                            // not in skip table, add it
                                            skip = new HarvestSkipURI(source, STATE_CLASS, artifact.getURI(), releaseDate, this.errorMessage);
                                        }
                                        if (ArtifactHarvester.PROPRIETARY.equals(skip.errorMessage) || ArtifactHarvester.PROPRIETARY.equals(this.errorMessage)) {
                                            skip.setTryAfter(releaseDate);
                                            skip.errorMessage = errorMessage;
                                        }
                                        this.harvestSkipURIDAO.put(skip);
                                        this.downloadCount++;
                                        added = true;
                                        if (skip != null) {
                                            this.downloadCount--;
                                            if (ArtifactHarvester.PROPRIETARY.equals(this.errorMessage)) {
                                                this.updateCount++;
                                                message = this.errorMessage + " artifact already exists in skip table, update tryAfter date to release date.";
                                            } else {
                                                added = false;
                                                String msg = "artifact already exists in skip table.";
                                                ;
                                                if (this.reason.equalsIgnoreCase("None")) {
                                                    this.reason = "Public " + msg;
                                                } else {
                                                    this.reason = this.reason + " and public " + msg;
                                                }
                                            }
                                        }
                                    }
                                } catch (Exception ex) {
                                    success = false;
                                    message = "Failed to determine if artifact " + artifact.getURI() + " exists: " + ex.getMessage();
                                    log.error(message, ex);
                                }
                                logEnd(format(state.curID), artifact, success, added, message);
                            }
                        }
                    }
                }
                this.harvestStateDAO.put(state);
                log.debug("Updated artifact harvest state.  Date: " + state.curLastModified);
                log.debug("Updated artifact harvest state.  ID: " + format(state.curID));
                this.observationDAO.getTransactionManager().commitTransaction();
            } catch (Throwable t) {
                this.observationDAO.getTransactionManager().rollbackTransaction();
                throw t;
            }
        }
        return (observationStates.size() < batchSize + 1);
    } finally {
        logBatchEnd();
    }
}

Example 4 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class ArtifactValidator method getLogicalMetadata.

private TreeSet<ArtifactMetadata> getLogicalMetadata() throws Exception {
    TreeSet<ArtifactMetadata> result = new TreeSet<>(ArtifactMetadata.getComparator());
    if (StringUtil.hasText(source)) {
        // use database <server.database.schema>
        // HarvestSkipURI table is not supported in 'diff' mode, i.e. reportOnly = true
        this.supportSkipURITable = !reportOnly;
        long t1 = System.currentTimeMillis();
        List<ObservationState> states = observationDAO.getObservationList(collection, null, null, null);
        long t2 = System.currentTimeMillis();
        long dt = t2 - t1;
        log.info("get-state-list: size=" + states.size() + " in " + dt + " ms");
        int depth = 3;
        ListIterator<ObservationState> iter = states.listIterator();
        t1 = System.currentTimeMillis();
        while (iter.hasNext()) {
            ObservationState s = iter.next();
            // GC
            iter.remove();
            ObservationResponse resp = observationDAO.getObservationResponse(s, depth);
            if (resp == null) {
                log.error("Null response from Observation DAO, ObservationURI: " + s.getURI().toString() + ", depth: " + depth);
            } else if (resp.observation == null) {
                log.error("Observation is null, ObservationURI: " + s.getURI().toString() + ", depth: " + depth);
            } else {
                for (Plane plane : resp.observation.getPlanes()) {
                    for (Artifact artifact : plane.getArtifacts()) {
                        String observationID = s.getURI().getObservationID();
                        result.add(getMetadata(observationID, artifact, plane.dataRelease, plane.metaRelease));
                    }
                }
            }
        }
        log.info("Finished logical metadata query in " + (System.currentTimeMillis() - t1) + " ms");
    } else {
        this.supportSkipURITable = false;
        if (caomTapResourceID != null) {
            // source is a TAP resource ID
            AuthMethod authMethod = AuthenticationUtil.getAuthMethodFromCredentials(AuthenticationUtil.getCurrentSubject());
            TapClient tapClient = new TapClient(caomTapResourceID);
            try {
                this.caomTapURL = tapClient.getSyncURL(authMethod);
            } catch (ResourceNotFoundException ex) {
                if (ex.getMessage().contains("with password")) {
                    throw new ResourceNotFoundException("TAP service for " + caomTapResourceID + " does not support password authentication.", ex);
                }
            }
        }
        // source is a TAP service URL or a TAP resource ID
        String adql = "select distinct(a.uri), a.contentChecksum, a.contentLength, a.contentType, o.observationID, " + "a.productType, a.releaseType, p.dataRelease, p.metaRelease " + "from caom2.Artifact a " + "join caom2.Plane p on a.planeID = p.planeID " + "join caom2.Observation o on p.obsID = o.obsID " + "where o.collection='" + collection + "'";
        log.debug("logical query: " + adql);
        long start = System.currentTimeMillis();
        result = query(caomTapURL, adql);
        log.info("Finished caom2 query in " + (System.currentTimeMillis() - start) + " ms");
    }
    return result;
}

Example 5 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class GetActionTest method testDoIt.

@Test
public void testDoIt() throws Exception {
    // test the doIt method when it returns 2 observations
    HttpServletRequest mockRequest = mock(HttpServletRequest.class);
    GetAction getAction = new TestGetAction(mockDao);
    TestSyncOutput out = new TestSyncOutput();
    getAction.setSyncOutput(out);
    reset(mockDao);
    expect(mockRequest.getMethod()).andReturn("GET");
    expect(mockRequest.getServletPath()).andReturn("/obs").atLeastOnce();
    expect(mockRequest.getPathInfo()).andReturn("TEST").atLeastOnce();
    // build the list of observations for the mock dao to return
    List<ObservationState> obsList = new ArrayList<ObservationState>();
    DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC);
    ObservationState os1 = new ObservationState(new ObservationURI("TEST", "1234"));
    os1.maxLastModified = df.parse("2010-10-10T10:10:10.10");
    os1.accMetaChecksum = URI.create("md5:5b71d023d4729575d550536dce8439e6");
    obsList.add(os1);
    ObservationState os2 = new ObservationState(new ObservationURI("TEST", "6789"));
    os2.maxLastModified = df.parse("2011-11-11T11:11:11.111");
    os2.accMetaChecksum = URI.create("md5:aedbcf5e27a17fc2daa5a0e0d7840009");
    obsList.add(os2);
    Enumeration<String> params = Collections.emptyEnumeration();
    expect(mockRequest.getParameterNames()).andReturn(params);
    // since no maxRec argument given, expect the default one
    expect(mockDao.getObservationList("TEST", null, null, RepoAction.MAX_LIST_SIZE, true)).andReturn(obsList);
    replay(mockDao, mockRequest);
    getAction.setSyncInput(new SyncInput(mockRequest, getAction.getInlineContentHandler()));
    getAction.run();
    String expected = "TEST" + "\t" + "1234" + "\t" + df.format(os1.maxLastModified) + "\t" + os1.accMetaChecksum.toString() + "\n" + "TEST" + "\t" + "6789" + "\t" + df.format(os2.maxLastModified) + "\t" + os2.accMetaChecksum.toString() + "\n";
    String content = out.getContent();
    log.debug("\n--list content start--\n" + content + "\n--list content end--");
    Assert.assertEquals(expected, content);
    // repeat test when start, end, maxRec and ascendingOrder specified
    getAction = new TestGetAction(mockDao);
    reset(mockDao);
    reset(mockRequest);
    // get a new OutSync
    out = new TestSyncOutput();
    getAction.setSyncOutput(out);
    // build the list of observations for the mock dao to return
    expect(mockRequest.getMethod()).andReturn("GET");
    expect(mockRequest.getPathInfo()).andReturn("/TEST");
    List<String> keys = new ArrayList<String>();
    keys.add("MAXREC");
    keys.add("Start");
    keys.add("end");
    params = Collections.enumeration(keys);
    String startDate = "2010-10-10T10:10:10.1";
    String endDate = "2011-11-11T11:11:11.111";
    expect(mockRequest.getParameterNames()).andReturn(params);
    expect(mockRequest.getParameterValues("MAXREC")).andReturn(new String[] { "3" });
    expect(mockRequest.getParameterValues("Start")).andReturn(new String[] { startDate });
    expect(mockRequest.getParameterValues("end")).andReturn(new String[] { endDate });
    // all arguments given
    expect(mockDao.getObservationList("TEST", df.parse(startDate), df.parse(endDate), 3, true)).andReturn(obsList);
    replay(mockDao, mockRequest);
    getAction.setSyncInput(new SyncInput(mockRequest, getAction.getInlineContentHandler()));
    getAction.run();
    Assert.assertEquals(expected, out.getContent());
    log.info("Finished testDoIt.");
}