Examples with BlobAccessor com.pratilipi.data.BlobAccessor used on opensource projects

Example 1 with BlobAccessor

use of com.pratilipi.data.BlobAccessor in project pratilipi by Pratilipi.

the class DataStoreUtil method main.

public static void main(String... args) throws IOException, UnexpectedServerException, InterruptedException, ParseException {
    RemoteApiOptions options = new RemoteApiOptions().server("m.gamma.pratilipi.com", 80).useServiceAccountCredential("prod-pratilipi@appspot.gserviceaccount.com", "PrivateKey.p12").remoteApiPath("/remote_api");
    RemoteApiInstaller installer = new RemoteApiInstaller();
    installer.install(options);
    ObjectifyService.begin();
    Memcache memcache = DataAccessorFactory.getL2CacheAccessor();
    GcsService gcsService = GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
    DataAccessor dataAccessor = DataAccessorFactory.getDataAccessor();
    SearchAccessor searchAccessor = DataAccessorFactory.getSearchAccessor();
    BlobAccessor blobAccessor = DataAccessorFactory.getBlobAccessor();
    DocAccessor docAccessor = DataAccessorFactory.getDocAccessor();
    // START
    // END
    installer.uninstall();
}

Example 2 with BlobAccessor

use of com.pratilipi.data.BlobAccessor in project pratilipi by Pratilipi.

the class DataStoreBackupUtil method csvUser.

public static void csvUser() throws UnexpectedServerException {
    BlobAccessor blobAccessor = DataAccessorFactory.getBlobAccessorBackup();
    DateFormat csvDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
    csvDateFormat.setTimeZone(TimeZone.getTimeZone("Asia/Kolkata"));
    String CSV_HEADER = "UserId,FacebookId,Email,SignUpDate";
    String CSV_SEPARATOR = ",";
    String LINE_SEPARATOR = "\n";
    StringBuilder csv = new StringBuilder(CSV_HEADER + LINE_SEPARATOR);
    int count = 0;
    QueryResultIterator<UserEntity> itr = ObjectifyService.ofy().cache(false).load().type(UserEntity.class).chunk(1000).iterator();
    while (itr.hasNext()) {
        User user = itr.next();
        csv.append("'" + user.getId().toString()).append(CSV_SEPARATOR).append(user.getFacebookId() == null ? "" : "'" + user.getFacebookId()).append(CSV_SEPARATOR).append(user.getEmail() == null ? "" : user.getEmail()).append(CSV_SEPARATOR).append(csvDateFormat.format(user.getSignUpDate())).append(LINE_SEPARATOR);
        count++;
        if (count % 1000 == 0)
            System.out.println(count + " ...");
    }
    System.out.println(count + " ...");
    BlobEntry userCsvEntry = blobAccessor.newBlob("datastore/user.csv", csv.toString().getBytes(Charset.forName("UTF-8")), "text/csv");
    blobAccessor.createOrUpdateBlob(userCsvEntry);
}

Example 3 with BlobAccessor

use of com.pratilipi.data.BlobAccessor in project pratilipi by Pratilipi.

the class DataStoreBackupUtil method csvPratilipi.

public static void csvPratilipi() throws UnexpectedServerException {
    BlobAccessor blobAccessor = DataAccessorFactory.getBlobAccessorBackup();
    DateFormat csvDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
    csvDateFormat.setTimeZone(TimeZone.getTimeZone("Asia/Kolkata"));
    String CSV_HEADER = "PratilipiId,AuthorId," + "Title,TitleEN,Language,HasSummary," + "Type,ContentType,State,HasCover,ListingDate," + "ReviewCount, RatingCount, TotalRating, ReadCount";
    String CSV_SEPARATOR = ",";
    String LINE_SEPARATOR = "\n";
    StringBuilder csv = new StringBuilder(CSV_HEADER + LINE_SEPARATOR);
    int count = 0;
    QueryResultIterator<PratilipiEntity> itr = ObjectifyService.ofy().cache(false).load().type(PratilipiEntity.class).chunk(1000).iterator();
    while (itr.hasNext()) {
        Pratilipi pratilipi = itr.next();
        csv.append("'" + pratilipi.getId()).append(CSV_SEPARATOR).append(pratilipi.getAuthorId() == null ? "" : "'" + pratilipi.getAuthorId()).append(CSV_SEPARATOR).append(pratilipi.getTitle() == null ? "" : "\"" + pratilipi.getTitle().replace("\"", "\"\"") + "\"").append(CSV_SEPARATOR).append(pratilipi.getTitleEn() == null ? "" : "\"" + pratilipi.getTitleEn().replace("\"", "\"\"") + "\"").append(CSV_SEPARATOR).append(pratilipi.getLanguage()).append(CSV_SEPARATOR).append(pratilipi.getSummary() != null && pratilipi.getSummary().trim().length() != 0).append(CSV_SEPARATOR).append(pratilipi.getType()).append(CSV_SEPARATOR).append(pratilipi.getContentType()).append(CSV_SEPARATOR).append(pratilipi.getState()).append(CSV_SEPARATOR).append(pratilipi.getCoverImage() != null).append(CSV_SEPARATOR).append(csvDateFormat.format(pratilipi.getListingDate())).append(CSV_SEPARATOR).append(pratilipi.getReviewCount()).append(CSV_SEPARATOR).append(pratilipi.getRatingCount()).append(CSV_SEPARATOR).append(pratilipi.getTotalRating()).append(CSV_SEPARATOR).append(pratilipi.getReadCount()).append(LINE_SEPARATOR);
        count++;
        if (count % 1000 == 0)
            System.out.println(count + " ...");
    }
    System.out.println(count + " ...");
    BlobEntry authorCsvEntry = blobAccessor.newBlob("datastore/pratilipi.csv", csv.toString().getBytes(Charset.forName("UTF-8")), "text/csv");
    blobAccessor.createOrUpdateBlob(authorCsvEntry);
}

Example 4 with BlobAccessor

use of com.pratilipi.data.BlobAccessor in project pratilipi by Pratilipi.

the class UserBackupApi method get.

@Get
public GenericResponse get(GetRequest request) throws UnexpectedServerException {
    DataAccessor dataAccessor = DataAccessorFactory.getDataAccessor();
    BlobAccessor blobAccessor = DataAccessorFactory.getBlobAccessorBackup();
    Date backupDate = new Date();
    DateFormat yearFormat = new SimpleDateFormat("yyyy");
    DateFormat dayFormat = new SimpleDateFormat("dd");
    DateFormat hourFormat = new SimpleDateFormat("HH");
    DateFormat csvDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
    yearFormat.setTimeZone(TimeZone.getTimeZone("Asia/Kolkata"));
    dayFormat.setTimeZone(TimeZone.getTimeZone("Asia/Kolkata"));
    hourFormat.setTimeZone(TimeZone.getTimeZone("Asia/Kolkata"));
    csvDateFormat.setTimeZone(TimeZone.getTimeZone("Asia/Kolkata"));
    StringBuilder backup = new StringBuilder();
    StringBuilder csv = new StringBuilder(CSV_HEADER + LINE_SEPARATOR);
    Gson gson = new GsonBuilder().registerTypeAdapter(Date.class, new GsonLongDateAdapter()).create();
    int batchSize = 1000;
    String cursor = null;
    int count = 0;
    while (true) {
        DataListCursorTuple<User> userListCursorTupe = dataAccessor.getUserList(cursor, batchSize);
        List<User> userList = userListCursorTupe.getDataList();
        for (User user : userList) {
            backup.append(gson.toJson(user) + LINE_SEPARATOR);
            if (request.generateCsv())
                csv.append("'" + user.getId().toString()).append(CSV_SEPARATOR).append(user.getFacebookId() == null ? "" : "'" + user.getFacebookId()).append(CSV_SEPARATOR).append(user.getEmail() == null ? "" : user.getEmail()).append(CSV_SEPARATOR).append(csvDateFormat.format(user.getSignUpDate())).append(LINE_SEPARATOR);
        }
        count = count + userList.size();
        if (userList.size() < batchSize)
            break;
        else
            cursor = userListCursorTupe.getCursor();
    }
    String year = yearFormat.format(backupDate);
    String day = dayFormat.format(backupDate);
    String hour = hourFormat.format(backupDate);
    String fileName = "datastore.user/" + year + "-mm-" + day + "/" + "user-" + year + "-mm-" + day + "-" + hour + ":xx-IST";
    BlobEntry userBackupEntry = blobAccessor.newBlob(fileName, backup.toString().getBytes(Charset.forName("UTF-8")), "text/plain");
    blobAccessor.createOrUpdateBlob(userBackupEntry);
    if (request.generateCsv()) {
        BlobEntry userCsvEntry = blobAccessor.newBlob("datastore/user.csv", csv.toString().getBytes(Charset.forName("UTF-8")), "text/csv");
        blobAccessor.createOrUpdateBlob(userCsvEntry);
    }
    logger.log(Level.INFO, "Backed up " + count + " User Entities.");
    return new GenericResponse();
}

Example 5 with BlobAccessor

use of com.pratilipi.data.BlobAccessor in project pratilipi by Pratilipi.

the class PratilipiDocUtil method updatePratilipiContent.

public static void updatePratilipiContent(Long pratilipiId) throws UnexpectedServerException {
    DataAccessor dataAccessor = DataAccessorFactory.getDataAccessor();
    BlobAccessor blobAccessor = DataAccessorFactory.getBlobAccessor();
    DocAccessor docAccessor = DataAccessorFactory.getDocAccessor();
    PratilipiContentDoc pcDoc = DataAccessorFactory.getDocAccessor().newPratilipiContentDoc();
    Pratilipi pratilipi = dataAccessor.getPratilipi(pratilipiId);
    if (!pratilipi.isOldContent()) {
        return;
    } else if (pratilipi.getContentType() == PratilipiContentType.PRATILIPI) {
        BlobEntry blobEntry = blobAccessor.getBlob("pratilipi-content/pratilipi/" + pratilipiId);
        if (blobEntry == null)
            return;
        String contentHtml = new String(blobEntry.getData(), Charset.forName("UTF-8"));
        List<Object[]> pageletList = _createPageletList(pratilipi, Jsoup.parse(contentHtml).body());
        if (pageletList.size() > 0) {
            PratilipiContentDoc.Chapter chapter = null;
            if (pageletList.get(0)[0] != PratilipiContentDoc.PageletType.HEAD)
                chapter = pcDoc.addChapter(pratilipi.getTitle() == null ? pratilipi.getTitleEn() : pratilipi.getTitle());
            for (Object[] pagelet : pageletList) {
                if (pagelet[0] == PratilipiContentDoc.PageletType.HEAD) {
                    chapter = pcDoc.addChapter((String) pagelet[1]);
                } else {
                    PratilipiContentDoc.Page page = chapter.getPage(1);
                    if (page == null)
                        page = chapter.addPage();
                    page.addPagelet((PratilipiContentDoc.PageletType) pagelet[0], pagelet[1], (PratilipiContentDoc.AlignmentType) pagelet[2]);
                }
            }
        }
    } else if (pratilipi.getContentType() == PratilipiContentType.IMAGE) {
        for (int i = 1; i <= pratilipi.getPageCount(); i++) {
            BlobEntry blobEntry = blobAccessor.getBlob("pratilipi/" + pratilipiId + "/images/" + i);
            if (pratilipi.getId() == 5639838220943360L && i <= 5)
                // Skipping first 5 pages as per Shally's request
                continue;
            else if (pratilipi.getId() == 5749258686824448L && i <= 4)
                // Skipping first 4 pages as per Shally's request
                continue;
            else if (pratilipi.getId() == 5486454792781824L && i <= 1)
                // Skipping first page as per Shally's request
                continue;
            else if (blobEntry == null && pratilipi.getId() == 5768181499035648L)
                // Skipping missing pages as per Dileepan's request
                continue;
            JsonObject imgData = new JsonObject();
            imgData.addProperty("name", i + "");
            imgData.addProperty("height", ImageUtil.getHeight(blobEntry));
            imgData.addProperty("width", ImageUtil.getWidth(blobEntry));
            pcDoc.addChapter(null).addPage().addPagelet(PratilipiContentDoc.PageletType.IMAGE, imgData);
        }
    } else {
        throw new UnexpectedServerException("ContentType " + pratilipi.getContentType() + " is not supported !");
    }
    docAccessor.save(pratilipiId, pcDoc);
}