Examples with Cursor com.google.appengine.api.datastore.Cursor used on opensource projects

Example 11 with Cursor

use of com.google.appengine.api.datastore.Cursor in project pratilipi by Pratilipi.

the class PratilipiBackupApi method get.

@Get
public GenericResponse get(GetRequest request) throws UnexpectedServerException {
    DataAccessor dataAccessor = DataAccessorFactory.getDataAccessor();
    // Fetching Cursor from AppProperty
    AppProperty appProperty = dataAccessor.getAppProperty(AppProperty.API_PRATILIPI_BACKUP);
    if (appProperty == null)
        appProperty = dataAccessor.newAppProperty(AppProperty.API_PRATILIPI_BACKUP);
    Cursor cursor = appProperty.getValue() == null ? null : Cursor.fromWebSafeString((String) appProperty.getValue());
    QueryResultIterator<Key<PratilipiEntity>> itr = ObjectifyService.ofy().load().type(PratilipiEntity.class).filter("LAST_UPDATED >=", // Thu Dec 01 00:00:00 IST 2016
    new Date(1480530600000L)).order("LAST_UPDATED").startAt(cursor).keys().iterator();
    int batchSize = 1000;
    List<Task> taskList = new ArrayList<>(batchSize);
    while (itr.hasNext()) {
        TaskQueueFactory.getPratilipiTaskQueue().add(TaskQueueFactory.newTask().setUrl("/pratilipi/backup").addParam("pratilipiId", itr.next().getId() + ""));
        if (taskList.size() == batchSize || !itr.hasNext()) {
            TaskQueueFactory.getPratilipiTaskQueue().addAll(taskList);
            taskList.clear();
        }
    }
    // Updating Cursor to AppProperty
    appProperty.setValue(itr.getCursor().toWebSafeString());
    dataAccessor.createOrUpdateAppProperty(appProperty);
    /*		DataAccessor dataAccessor = DataAccessorFactory.getDataAccessor();
		BlobAccessor blobAccessor = DataAccessorFactory.getBlobAccessorBackup();
		
		Date backupDate = new Date();

		DateFormat dateFormat = new SimpleDateFormat( "yyyy-MM-dd" );
		DateFormat csvDateFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm" );
		DateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd-HH:mm-z" );
		dateFormat.setTimeZone( TimeZone.getTimeZone( "Asia/Kolkata" ) );
		csvDateFormat.setTimeZone( TimeZone.getTimeZone( "Asia/Kolkata" ) );
		dateTimeFormat.setTimeZone( TimeZone.getTimeZone( "Asia/Kolkata" ) );

		StringBuilder backup = new StringBuilder();
		StringBuilder csv = new StringBuilder( CSV_HEADER + LINE_SEPARATOR );

		int count = 0;
		String cursor = null;
		PratilipiFilter pratilipiFilter = new PratilipiFilter();
		Gson gson = new GsonBuilder().registerTypeAdapter( Date.class, new GsonIstDateAdapter() ).create();
		
		while( true ) {
			DataListCursorTuple<Pratilipi> pratilipiListCursorTuple = dataAccessor.getPratilipiList( pratilipiFilter, cursor, 1000 );
			List<Pratilipi> pratilipiList = pratilipiListCursorTuple.getDataList();

			for( Pratilipi pratilipi : pratilipiList ) {
                backup.append( gson.toJson( pratilipi ) + LINE_SEPARATOR );

				if( request.generateCsv != null && request.generateCsv )
					csv.append( "'" + pratilipi.getId() )
							.append( CSV_SEPARATOR ).append( pratilipi.getAuthorId() == null ? "" : "'" + pratilipi.getAuthorId() )
							.append( CSV_SEPARATOR ).append( pratilipi.getTitle()	 == null ? "" : "\"" + pratilipi.getTitle().replace( "\"", "\"\"" ) + "\"" )
							.append( CSV_SEPARATOR ).append( pratilipi.getTitleEn()	 == null ? "" : "\"" + pratilipi.getTitleEn().replace( "\"", "\"\"" ) + "\"" )
							.append( CSV_SEPARATOR ).append( pratilipi.getLanguage() )
							.append( CSV_SEPARATOR ).append( pratilipi.getSummary() != null && pratilipi.getSummary().trim().length() != 0 )
							.append( CSV_SEPARATOR ).append( pratilipi.getType() )
							.append( CSV_SEPARATOR ).append( pratilipi.getContentType() )
							.append( CSV_SEPARATOR ).append( pratilipi.getState() )
							.append( CSV_SEPARATOR ).append( pratilipi.getCoverImage() != null )
							.append( CSV_SEPARATOR ).append( csvDateFormat.format( pratilipi.getListingDate() ) )
							.append( CSV_SEPARATOR ).append( pratilipi.getReviewCount() )
							.append( CSV_SEPARATOR ).append( pratilipi.getRatingCount() )
							.append( CSV_SEPARATOR ).append( pratilipi.getTotalRating() )
							.append( CSV_SEPARATOR ).append( pratilipi.getReadCount() )
							.append( LINE_SEPARATOR );
				
			}
			
			count = count + pratilipiList.size();

			if( pratilipiList.size() < 1000 )
				break;
			else
				cursor = pratilipiListCursorTuple.getCursor();
		}
		

		String fileName = "datastore.pratilipi/"
				+ dateFormat.format( backupDate ) + "/"
				+ "pratilipi-" + dateTimeFormat.format( backupDate );

		BlobEntry pratilipiBackupEntry = blobAccessor.newBlob(
				fileName,
				backup.toString().getBytes( Charset.forName( "UTF-8" ) ),
				"text/plain" );
		
		blobAccessor.createOrUpdateBlob( pratilipiBackupEntry );
		
		
		if( request.generateCsv != null && request.generateCsv ) {
			BlobEntry authorCsvEntry = blobAccessor.newBlob(
					"datastore/pratilipi.csv",
					csv.toString().getBytes( Charset.forName( "UTF-8" ) ),
					"text/csv" );
			blobAccessor.createOrUpdateBlob( authorCsvEntry );
		}
		

		logger.log( Level.INFO, "Backed up " + count + " Pratilipi Entities." );
*/
    return new GenericResponse();
}

Example 12 with Cursor

use of com.google.appengine.api.datastore.Cursor in project siena by mandubian.

the class GaePersistenceManagerAsync method doFetchIterable.

private <T> SienaFuture<Iterable<T>> doFetchIterable(QueryAsync<T> query, int limit, int offset) {
    QueryOptionGaeContext gaeCtx = (QueryOptionGaeContext) query.option(QueryOptionGaeContext.ID);
    QueryOptionState state = (QueryOptionState) query.option(QueryOptionState.ID);
    QueryOptionFetchType fetchType = (QueryOptionFetchType) query.option(QueryOptionFetchType.ID);
    if (gaeCtx == null) {
        gaeCtx = new QueryOptionGaeContext();
        query.customize(gaeCtx);
    }
    FetchOptions fetchOptions = FetchOptions.Builder.withDefaults();
    QueryOptionPage pag = (QueryOptionPage) query.option(QueryOptionPage.ID);
    if (!pag.isPaginating()) {
        // no pagination but pageOption active
        if (pag.isActive()) {
            // if local limit is set, it overrides the pageOption.pageSize
            if (limit != Integer.MAX_VALUE) {
                gaeCtx.realPageSize = limit;
                fetchOptions.limit(gaeCtx.realPageSize);
                // pageOption is passivated to be sure it is not reused
                pag.passivate();
            } else // using pageOption.pageSize
            {
                gaeCtx.realPageSize = pag.pageSize;
                fetchOptions.limit(gaeCtx.realPageSize);
                // passivates the pageOption in stateless mode not to keep anything between 2 requests
                if (state.isStateless()) {
                    pag.passivate();
                }
            }
        } else {
            if (limit != Integer.MAX_VALUE) {
                gaeCtx.realPageSize = limit;
                fetchOptions.limit(gaeCtx.realPageSize);
            }
        }
    } else {
        // paginating so use the pagesize and don't passivate pageOption
        // local limit is not taken into account
        gaeCtx.realPageSize = pag.pageSize;
        fetchOptions.limit(gaeCtx.realPageSize);
    }
    QueryOptionOffset off = (QueryOptionOffset) query.option(QueryOptionOffset.ID);
    // if local offset has been set, uses it
    if (offset != 0) {
        off.activate();
        off.offset = offset;
    }
    // if previousPage has detected there is no more data, simply returns an empty list
    if (gaeCtx.noMoreDataBefore) {
        return new SienaFutureMock<Iterable<T>>(new ArrayList<T>());
    }
    if (state.isStateless()) {
        if (pag.isPaginating()) {
            if (off.isActive()) {
                gaeCtx.realOffset += off.offset;
                fetchOptions.offset(gaeCtx.realOffset);
                off.passivate();
            } else {
                fetchOptions.offset(gaeCtx.realOffset);
            }
        } else {
            // if stateless and not paginating, resets the realoffset to 0
            gaeCtx.realOffset = off.offset;
            if (off.isActive()) {
                fetchOptions.offset(gaeCtx.realOffset);
                off.passivate();
            }
        }
        switch(fetchType.fetchType) {
            case ITER:
            default:
                {
                    // uses iterable as it is the only async request for prepared query for the time being
                    Iterable<Entity> entities = prepare(query).asIterable(fetchOptions);
                    return new GaeSienaFutureIterableMapper<T>(this, entities, query);
                }
        }
    } else {
        if (off.isActive()) {
            // by default, we add the offset but it can be added with the realoffset 
            // in case of cursor desactivated
            fetchOptions.offset(off.offset);
            gaeCtx.realOffset += off.offset;
            off.passivate();
        }
        // manages cursor limitations for IN and != operators		
        if (!gaeCtx.isActive()) {
            // cursor not yet created
            switch(fetchType.fetchType) {
                case ITER:
                default:
                    {
                        PreparedQuery pq = prepare(query);
                        if (pag.isPaginating()) {
                            // in case of pagination, we need to allow asynchronous calls such as:
                            // QueryAsync<MyClass> query = pm.createQuery(MyClass).paginate(5).stateful().order("name");
                            // SienaFuture<Iterable<MyClass>> future1 = query.iter();
                            // SienaFuture<Iterable<MyClass>> future2 = query.nextPage().iter();
                            // Iterable<MyClass> it = future1.get().iterator();
                            // while(it.hasNext()) { // do it }
                            // it = future2.get().iterator();
                            // while(it.hasNext()) { // do it }
                            // so we can't use the asQueryResultIterable as the cursor is not moved to the end of the current page
                            // but moved at each call of iterable.iterator().next()
                            // thus we use the List in this case to be able to move directly to the next page with cursors
                            QueryResultList<Entity> entities = pq.asQueryResultList(fetchOptions);
                            // activates the GaeCtx now that it is initialised
                            gaeCtx.activate();
                            // sets the current cursor (in stateful mode, cursor is always kept for further use)
                            //if(gaeCtx.useCursor){
                            Cursor cursor = entities.getCursor();
                            if (cursor != null) {
                                gaeCtx.addCursor(cursor.toWebSafeString());
                            }
                            return new GaeSienaFutureIterableMapper<T>(this, entities, query);
                        } else {
                            // if not paginating, we simply use the queryresultiterable and moves the current cursor
                            // while iterating
                            QueryResultIterable<Entity> entities = pq.asQueryResultIterable(fetchOptions);
                            // activates the GaeCtx now that it is initialised
                            gaeCtx.activate();
                            return new GaeSienaFutureIterableMapperWithCursor<T>(this, entities, query);
                        }
                    }
            }
        } else {
            switch(fetchType.fetchType) {
                case ITER:
                default:
                    {
                        PreparedQuery pq = prepare(query);
                        if (pag.isPaginating()) {
                            // in case of pagination, we need to allow asynchronous calls such as:
                            // QueryAsync<MyClass> query = pm.createQuery(MyClass).paginate(5).stateful().order("name");
                            // SienaFuture<Iterable<MyClass>> future1 = query.iter();
                            // SienaFuture<Iterable<MyClass>> future2 = query.nextPage().iter();
                            // Iterable<MyClass> it = future1.get().iterator();
                            // while(it.hasNext()) { // do it }
                            // it = future2.get().iterator();
                            // while(it.hasNext()) { // do it }
                            // so we can't use the asQueryResultIterable as the cursor is not moved to the end of the current page
                            // but moved at each call of iterable.iterator().next()
                            // thus we use the List in this case to be able to move directly to the next page with cursors
                            QueryResultList<Entity> entities;
                            if (!gaeCtx.useCursor) {
                                // then uses offset (in case of IN or != operators)
                                //if(offset.isActive()){
                                //	fetchOptions.offset(offset.offset);
                                //}
                                fetchOptions.offset(gaeCtx.realOffset);
                                entities = pq.asQueryResultList(fetchOptions);
                            } else {
                                String cursor = gaeCtx.currentCursor();
                                if (cursor != null) {
                                    entities = pq.asQueryResultList(fetchOptions.startCursor(Cursor.fromWebSafeString(cursor)));
                                } else {
                                    entities = pq.asQueryResultList(fetchOptions);
                                }
                                // sets the current cursor (in stateful mode, cursor is always kept for further use)
                                //if(gaeCtx.useCursor){
                                gaeCtx.addCursor(entities.getCursor().toWebSafeString());
                            //}
                            }
                            return new GaeSienaFutureIterableMapper<T>(this, entities, query);
                        } else {
                            // if not paginating, we simply use the queryresultiterable and moves the current cursor
                            // while iterating
                            QueryResultIterable<Entity> entities;
                            if (!gaeCtx.useCursor) {
                                // then uses offset (in case of IN or != operators)
                                //if(offset.isActive()){
                                //	fetchOptions.offset(offset.offset);
                                //}
                                fetchOptions.offset(gaeCtx.realOffset);
                                entities = pq.asQueryResultIterable(fetchOptions);
                            } else {
                                String cursor = gaeCtx.currentCursor();
                                if (cursor != null) {
                                    entities = pq.asQueryResultIterable(fetchOptions.startCursor(Cursor.fromWebSafeString(gaeCtx.currentCursor())));
                                } else {
                                    entities = pq.asQueryResultIterable(fetchOptions);
                                }
                            }
                            return new GaeSienaFutureIterableMapperWithCursor<T>(this, entities, query);
                        }
                    }
            }
        }
    }
}

Example 13 with Cursor

use of com.google.appengine.api.datastore.Cursor in project pratilipi by Pratilipi.

the class DataAccessorGaeImpl method getUserAuthorList.

@Override
public DataListCursorTuple<UserAuthor> getUserAuthorList(Long userId, Long authorId, String cursorStr, Integer offset, Integer resultCount) {
    Query<UserAuthorEntity> query = ObjectifyService.ofy().load().type(UserAuthorEntity.class);
    if (userId != null)
        query = query.filter("USER_ID", userId);
    if (authorId != null)
        query = query.filter("AUTHOR_ID", authorId);
    if (cursorStr != null)
        query = query.startAt(Cursor.fromWebSafeString(cursorStr));
    if (offset != null && offset > 0)
        query = query.offset(offset);
    if (resultCount != null && resultCount > 0)
        query = query.limit(resultCount);
    QueryResultIterator<UserAuthorEntity> iterator = query.iterator();
    // UserAuthor List
    ArrayList<UserAuthor> userAuthorList = resultCount == null ? new ArrayList<UserAuthor>() : new ArrayList<UserAuthor>(resultCount);
    while (iterator.hasNext()) userAuthorList.add(iterator.next());
    // Cursor
    Cursor cursor = iterator.getCursor();
    return new DataListCursorTuple<UserAuthor>(userAuthorList, cursor == null ? null : cursor.toWebSafeString());
}

Example 14 with Cursor

use of com.google.appengine.api.datastore.Cursor in project pratilipi by Pratilipi.

the class DataAccessorGaeImpl method getUserList.

@SuppressWarnings("unchecked")
private <T> DataListCursorTuple<T> getUserList(String cursorStr, Integer resultCount, boolean idOnly) {
    Query<UserEntity> query = ObjectifyService.ofy().load().type(UserEntity.class);
    if (cursorStr != null)
        query = query.startAt(Cursor.fromWebSafeString(cursorStr));
    if (resultCount != null && resultCount > 0)
        query = query.limit(resultCount);
    List<T> dataList = resultCount == null ? new ArrayList<T>() : new ArrayList<T>(resultCount);
    Cursor cursor = null;
    if (idOnly) {
        QueryResultIterator<Key<UserEntity>> iterator = query.keys().iterator();
        while (iterator.hasNext()) dataList.add((T) (Long) iterator.next().getId());
        cursor = iterator.getCursor();
    } else {
        QueryResultIterator<UserEntity> iterator = query.iterator();
        while (iterator.hasNext()) dataList.add((T) iterator.next());
        cursor = iterator.getCursor();
    }
    return new DataListCursorTuple<T>(dataList, cursor == null ? null : cursor.toWebSafeString());
}