Examples with NoteInfo org.apache.zeppelin.notebook.NoteInfo used on opensource projects

Example 1 with NoteInfo

use of org.apache.zeppelin.notebook.NoteInfo in project zeppelin by apache.

the class NotebookRepoSync method sync.

/**
   * Copies new/updated notes from source to destination storage
   *
   * @throws IOException
   */
void sync(int sourceRepoIndex, int destRepoIndex, AuthenticationInfo subject) throws IOException {
    LOG.info("Sync started");
    NotebookAuthorization auth = NotebookAuthorization.getInstance();
    NotebookRepo srcRepo = getRepo(sourceRepoIndex);
    NotebookRepo dstRepo = getRepo(destRepoIndex);
    List<NoteInfo> allSrcNotes = srcRepo.list(subject);
    List<NoteInfo> srcNotes = auth.filterByUser(allSrcNotes, subject);
    List<NoteInfo> dstNotes = dstRepo.list(subject);
    Map<String, List<String>> noteIds = notesCheckDiff(srcNotes, srcRepo, dstNotes, dstRepo, subject);
    List<String> pushNoteIds = noteIds.get(pushKey);
    List<String> pullNoteIds = noteIds.get(pullKey);
    List<String> delDstNoteIds = noteIds.get(delDstKey);
    if (!pushNoteIds.isEmpty()) {
        LOG.info("Notes with the following IDs will be pushed");
        for (String id : pushNoteIds) {
            LOG.info("ID : " + id);
        }
        pushNotes(subject, pushNoteIds, srcRepo, dstRepo, false);
    } else {
        LOG.info("Nothing to push");
    }
    if (!pullNoteIds.isEmpty()) {
        LOG.info("Notes with the following IDs will be pulled");
        for (String id : pullNoteIds) {
            LOG.info("ID : " + id);
        }
        pushNotes(subject, pullNoteIds, dstRepo, srcRepo, true);
    } else {
        LOG.info("Nothing to pull");
    }
    if (!delDstNoteIds.isEmpty()) {
        LOG.info("Notes with the following IDs will be deleted from dest");
        for (String id : delDstNoteIds) {
            LOG.info("ID : " + id);
        }
        deleteNotes(subject, delDstNoteIds, dstRepo);
    } else {
        LOG.info("Nothing to delete from dest");
    }
    LOG.info("Sync ended");
}

Example 2 with NoteInfo

use of org.apache.zeppelin.notebook.NoteInfo in project zeppelin by apache.

the class S3NotebookRepo method list.

@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
    List<NoteInfo> infos = new LinkedList<>();
    NoteInfo info;
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (objectSummary.getKey().endsWith("note.json")) {
                    info = getNoteInfo(objectSummary.getKey());
                    if (info != null) {
                        infos.add(info);
                    }
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to list objects in S3: " + ace, ace);
    }
    return infos;
}

Example 3 with NoteInfo

use of org.apache.zeppelin.notebook.NoteInfo in project zeppelin by apache.

the class MongoNotebookRepo method list.

@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
    syncId();
    List<NoteInfo> infos = new LinkedList<>();
    MongoCursor<Document> cursor = coll.find().iterator();
    while (cursor.hasNext()) {
        Document doc = cursor.next();
        Note note = documentToNote(doc);
        NoteInfo info = new NoteInfo(note);
        infos.add(info);
    }
    cursor.close();
    return infos;
}

Example 4 with NoteInfo

use of org.apache.zeppelin.notebook.NoteInfo in project zeppelin by apache.

the class MongoNotebookRepo method insertFileSystemNotes.

/**
   * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true,
   * this method will insert local notes into MongoDB on startup.
   * If a note already exists in MongoDB, skip it.
   */
private void insertFileSystemNotes() throws IOException {
    // docs to be imported
    LinkedList<Document> docs = new LinkedList<>();
    NotebookRepo vfsRepo = new VFSNotebookRepo(this.conf);
    List<NoteInfo> infos = vfsRepo.list(null);
    // collect notes to be imported
    for (NoteInfo info : infos) {
        Note note = vfsRepo.get(info.getId(), null);
        Document doc = noteToDocument(note);
        docs.add(doc);
    }
    /*
     * 'ordered(false)' option allows to proceed bulk inserting even though
     * there are duplicated documents. The duplicated documents will be skipped
     * and print a WARN log.
     */
    try {
        coll.insertMany(docs, new InsertManyOptions().ordered(false));
    } catch (MongoBulkWriteException e) {
        //print duplicated document warning log
        printDuplicatedException(e);
    }
    // it does nothing for now but maybe in the future...
    vfsRepo.close();
}

Example 5 with NoteInfo

use of org.apache.zeppelin.notebook.NoteInfo in project zeppelin by apache.

the class NotebookRepoSync method notesCheckDiff.

private Map<String, List<String>> notesCheckDiff(List<NoteInfo> sourceNotes, NotebookRepo sourceRepo, List<NoteInfo> destNotes, NotebookRepo destRepo, AuthenticationInfo subject) {
    List<String> pushIDs = new ArrayList<>();
    List<String> pullIDs = new ArrayList<>();
    List<String> delDstIDs = new ArrayList<>();
    NoteInfo dnote;
    Date sdate, ddate;
    for (NoteInfo snote : sourceNotes) {
        dnote = containsID(destNotes, snote.getId());
        if (dnote != null) {
            try {
                /* note exists in source and destination storage systems */
                sdate = lastModificationDate(sourceRepo.get(snote.getId(), subject));
                ddate = lastModificationDate(destRepo.get(dnote.getId(), subject));
            } catch (IOException e) {
                LOG.error("Cannot access previously listed note {} from storage ", dnote.getId(), e);
                continue;
            }
            if (sdate.compareTo(ddate) != 0) {
                if (sdate.after(ddate) || oneWaySync) {
                    /* if source contains more up to date note - push
             * if oneWaySync is enabled, always push no matter who's newer */
                    pushIDs.add(snote.getId());
                    LOG.info("Modified note is added to push list : " + sdate);
                } else {
                    /* destination contains more up to date note - pull */
                    LOG.info("Modified note is added to pull list : " + ddate);
                    pullIDs.add(snote.getId());
                }
            }
        } else {
            /* note exists in source storage, and absent in destination
         * view source as up to date - push
         * (another scenario : note was deleted from destination - not considered)*/
            pushIDs.add(snote.getId());
        }
    }
    for (NoteInfo note : destNotes) {
        dnote = containsID(sourceNotes, note.getId());
        if (dnote == null) {
            /* note exists in destination storage, and absent in source */
            if (oneWaySync) {
                /* if oneWaySync is enabled, delete the note from destination */
                LOG.info("Extraneous note is added to delete dest list : " + note.getId());
                delDstIDs.add(note.getId());
            } else {
                /* if oneWaySync is disabled, pull the note from destination */
                LOG.info("Missing note is added to pull list : " + note.getId());
                pullIDs.add(note.getId());
            }
        }
    }
    Map<String, List<String>> map = new HashMap<>();
    map.put(pushKey, pushIDs);
    map.put(pullKey, pullIDs);
    map.put(delDstKey, delDstIDs);
    return map;
}