Search in sources :

Example 1 with AzureService

use of com.vmware.photon.controller.model.adapters.azure.model.cost.AzureService in project photon-model by vmware.

the class AzureCostStatsService method createServiceStatsForSubscription.

private void createServiceStatsForSubscription(Context context, AzureSubscription subscription) {
    Consumer<List<ComputeState>> serviceStatsProcessor = (subscriptionComputeStates) -> subscriptionComputeStates.forEach(subscriptionComputeState -> {
        List<ComputeStats> resourceStatsList = new ArrayList<>();
        ComputeStats subscriptionStats = new ComputeStats();
        subscriptionStats.statValues = new ConcurrentHashMap<>();
        subscriptionStats.computeLink = subscriptionComputeState.documentSelfLink;
        subscriptionStats.addCustomProperty(PhotonModelConstants.DOES_CONTAIN_SERVICE_STATS, Boolean.TRUE.toString());
        for (AzureService service : subscription.getServices().values()) {
            List<ComputeStats> resourcesCostStats = new ArrayList<>();
            Map<String, List<ServiceStat>> statsForAzureService = createStatsForAzureService(context, service, resourcesCostStats);
            subscriptionStats.statValues.putAll(statsForAzureService);
            resourceStatsList.addAll(resourcesCostStats);
        }
        if (!subscriptionStats.statValues.isEmpty()) {
            context.statsResponse.statsList.add(subscriptionStats);
        }
        if (!resourceStatsList.isEmpty()) {
            context.statsResponse.statsList.addAll(resourceStatsList);
        }
    });
    processSubscriptionStats(context, subscription, serviceStatsProcessor);
}
Also used : AzureCostConstants(com.vmware.photon.controller.model.adapters.azure.constants.AzureCostConstants) AzureStatsNormalizer(com.vmware.photon.controller.model.adapters.azure.utils.AzureStatsNormalizer) AuthCredentialsServiceState(com.vmware.xenon.services.common.AuthCredentialsService.AuthCredentialsServiceState) Arrays(java.util.Arrays) DateTimeZone(org.joda.time.DateTimeZone) ServiceTypeCluster(com.vmware.photon.controller.model.util.ClusterUtil.ServiceTypeCluster) QueryTask(com.vmware.xenon.services.common.QueryTask) ServiceDocument(com.vmware.xenon.common.ServiceDocument) AzureUriPaths(com.vmware.photon.controller.model.adapters.azure.AzureUriPaths) StringUtils(org.apache.commons.lang3.StringUtils) ComputeType(com.vmware.photon.controller.model.resources.ComputeDescriptionService.ComputeDescription.ComputeType) EndpointAllocationTaskService(com.vmware.photon.controller.model.tasks.EndpointAllocationTaskService) SingleResourceStatsCollectionTaskState(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState) Utils(com.vmware.xenon.common.Utils) BufferedSink(okio.BufferedSink) Map(java.util.Map) AzureService(com.vmware.photon.controller.model.adapters.azure.model.cost.AzureService) URI(java.net.URI) AzureSubscriptionsEnumerationRequest(com.vmware.photon.controller.model.adapters.azure.ea.enumeration.AzureSubscriptionsEnumerationService.AzureSubscriptionsEnumerationRequest) Builder(com.vmware.xenon.services.common.QueryTask.Builder) Path(java.nio.file.Path) EndpointState(com.vmware.photon.controller.model.resources.EndpointService.EndpointState) Interceptor(okhttp3.Interceptor) ComputeStatsResponse(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse) AzureDetailedBillHandler(com.vmware.photon.controller.model.adapters.azure.ea.utils.AzureDetailedBillHandler) Request(okhttp3.Request) ComputeStatsRequest(com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest) StatelessService(com.vmware.xenon.common.StatelessService) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) OldEaSummarizedBillElement(com.vmware.photon.controller.model.adapters.azure.model.cost.OldEaSummarizedBillElement) TaskManager(com.vmware.photon.controller.model.adapters.util.TaskManager) Set(java.util.Set) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) AzureSubscriptionsEnumerationService(com.vmware.photon.controller.model.adapters.azure.ea.enumeration.AzureSubscriptionsEnumerationService) AzureCostHelper(com.vmware.photon.controller.model.adapters.azure.ea.utils.AzureCostHelper) List(java.util.List) Stream(java.util.stream.Stream) UriUtils(com.vmware.xenon.common.UriUtils) Entry(java.util.Map.Entry) QueryOption(com.vmware.xenon.services.common.QueryTask.QuerySpecification.QueryOption) INTERNAL_REQUEST_TIMEOUT_SECONDS(com.vmware.photon.controller.model.adapters.azure.constants.AzureCostConstants.INTERNAL_REQUEST_TIMEOUT_SECONDS) ResourceMetrics(com.vmware.photon.controller.model.monitoring.ResourceMetricsService.ResourceMetrics) SingleResourceTaskCollectionStage(com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceTaskCollectionStage) Okio(okio.Okio) QueryByPages(com.vmware.photon.controller.model.query.QueryUtils.QueryByPages) OperationContext(com.vmware.xenon.common.OperationContext) AtomicDouble(com.google.common.util.concurrent.AtomicDouble) HashMap(java.util.HashMap) Function(java.util.function.Function) AzureResource(com.vmware.photon.controller.model.adapters.azure.model.cost.AzureResource) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) EndpointConfigRequest(com.vmware.photon.controller.model.adapterapi.EndpointConfigRequest) Query(com.vmware.xenon.services.common.QueryTask.Query) CollectionUtils(org.apache.commons.collections.CollectionUtils) EaBillLinkElement(com.vmware.photon.controller.model.adapters.azure.model.cost.EaBillLinkElement) ComputeState(com.vmware.photon.controller.model.resources.ComputeService.ComputeState) AUTO_DISCOVERED_ENTITY(com.vmware.photon.controller.model.constants.PhotonModelConstants.AUTO_DISCOVERED_ENTITY) EaBillLinks(com.vmware.photon.controller.model.adapters.azure.model.cost.EaBillLinks) BiConsumer(java.util.function.BiConsumer) Response(okhttp3.Response) EndpointType(com.vmware.photon.controller.model.constants.PhotonModelConstants.EndpointType) Call(okhttp3.Call) Callback(okhttp3.Callback) AzureSubscription(com.vmware.photon.controller.model.adapters.azure.model.cost.AzureSubscription) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) ExecutorService(java.util.concurrent.ExecutorService) OldApi(com.vmware.photon.controller.model.adapters.azure.model.cost.OldApi) AdapterUtils(com.vmware.photon.controller.model.adapters.util.AdapterUtils) ServiceErrorResponse(com.vmware.xenon.common.ServiceErrorResponse) Files(java.nio.file.Files) Operation(com.vmware.xenon.common.Operation) QueryUtils(com.vmware.photon.controller.model.query.QueryUtils) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) TypeName(com.vmware.xenon.common.ServiceDocumentDescription.TypeName) ServiceStat(com.vmware.xenon.common.ServiceStats.ServiceStat) File(java.io.File) QueryTop(com.vmware.photon.controller.model.query.QueryUtils.QueryTop) BillParsingStatus(com.vmware.photon.controller.model.adapters.azure.model.cost.BillParsingStatus) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) LocalDate(org.joda.time.LocalDate) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) OkHttpClient(okhttp3.OkHttpClient) Paths(java.nio.file.Paths) ClusterUtil(com.vmware.photon.controller.model.util.ClusterUtil) ComputeStateWithDescription(com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription) PhotonModelConstants(com.vmware.photon.controller.model.constants.PhotonModelConstants) ServiceDocumentDescription(com.vmware.xenon.common.ServiceDocumentDescription) Comparator(java.util.Comparator) QuerySpecification(com.vmware.xenon.services.common.QueryTask.QuerySpecification) OperationJoin(com.vmware.xenon.common.OperationJoin) ComputeStats(com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats) AzureService(com.vmware.photon.controller.model.adapters.azure.model.cost.AzureService) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList)

Example 2 with AzureService

use of com.vmware.photon.controller.model.adapters.azure.model.cost.AzureService in project photon-model by vmware.

the class AzureDetailedBillHandler method parseDetailedCsv.

public BillParsingStatus parseDetailedCsv(File billFile, Set<String> newSubscriptions, BillParsingStatus status, long billProcessedTimeMillis, String currency, BiConsumer<Map<String, AzureSubscription>, Long> dailyStatsConsumer) throws IOException {
    logger.fine(() -> "Beginning to parse CSV file.");
    try (CSVReader csvReader = new CSVReader(new FileReader(billFile), AzureCostConstants.DEFAULT_COLUMN_SEPARATOR, AzureCostConstants.DEFAULT_QUOTE_CHARACTER, AzureCostConstants.DEFAULT_ESCAPE_CHARACTER, (int) status.getNoLinesRead())) {
        HeaderColumnNameMappingStrategy<EaDetailedBillElement> strategy = new HeaderColumnNameMappingStrategy<>();
        strategy.setType(EaDetailedBillElement.class);
        long timeToStartBillProcessing = getTimeToStartBillProcessing(billProcessedTimeMillis);
        // This map will contain daily subscription, service & resource cost. The subscription
        // GUID is the key and the subscription details is the value. This map is maintained
        // since daily-level stats are needed for services and resources.
        Map<String, AzureSubscription> monthlyBill = new HashMap<>();
        String[] nextRow;
        Long prevRowEpoch = null;
        while ((nextRow = csvReader.readNext()) != null) {
            final String[] finalNextRow = nextRow;
            if (nextRow.length != BillHeaders.values().length) {
                // Skip any blank or malformed rows
                logger.warning(() -> String.format("Skipping malformed row: %s", Arrays.toString(finalNextRow)));
                continue;
            }
            logger.fine(() -> String.format("Beginning to process row: %s", Arrays.toString(finalNextRow)));
            EaDetailedBillElement detailedBillElement = AzureCostHelper.sanitizeDetailedBillElement(nextRow, currency);
            AzureSubscription subscription = populateSubscriptionCost(monthlyBill, detailedBillElement);
            long curRowEpoch = detailedBillElement.epochDate;
            if (shouldCreateServiceAndResourceCost(detailedBillElement, newSubscriptions, timeToStartBillProcessing)) {
                AzureService service = populateServiceCost(subscription, detailedBillElement);
                populateResourceCost(service, detailedBillElement);
            }
            billProcessedTimeMillis = billProcessedTimeMillis < curRowEpoch ? curRowEpoch : billProcessedTimeMillis;
            monthlyBill.put(detailedBillElement.subscriptionGuid, subscription);
            if (prevRowEpoch != null && !prevRowEpoch.equals(curRowEpoch)) {
                // This indicates that we have processed all rows belonging to a
                // corresponding day in the current month's bill.
                // Consume the batch
                // Subtract 1, to account for detecting date change line.
                status.setNoLinesRead(csvReader.getLinesRead() - 1);
                dailyStatsConsumer.accept(monthlyBill, null);
                break;
            }
            prevRowEpoch = curRowEpoch;
        }
        if ((nextRow == null && monthlyBill.size() > 0) || (nextRow == null && csvReader.getLinesRead() == AzureCostConstants.DEFAULT_LINES_TO_SKIP)) {
            status.setParsingComplete(true);
            dailyStatsConsumer.accept(monthlyBill, billProcessedTimeMillis);
            logger.fine(() -> "Finished parsing CSV bill.");
        }
        return status;
    }
}
Also used : CSVReader(com.opencsv.CSVReader) HashMap(java.util.HashMap) EaDetailedBillElement(com.vmware.photon.controller.model.adapters.azure.model.cost.EaDetailedBillElement) AzureSubscription(com.vmware.photon.controller.model.adapters.azure.model.cost.AzureSubscription) HeaderColumnNameMappingStrategy(com.opencsv.bean.HeaderColumnNameMappingStrategy) AzureService(com.vmware.photon.controller.model.adapters.azure.model.cost.AzureService) FileReader(java.io.FileReader)

Aggregations

AzureService (com.vmware.photon.controller.model.adapters.azure.model.cost.AzureService)2 AzureSubscription (com.vmware.photon.controller.model.adapters.azure.model.cost.AzureSubscription)2 HashMap (java.util.HashMap)2 Sets (com.google.common.collect.Sets)1 AtomicDouble (com.google.common.util.concurrent.AtomicDouble)1 CSVReader (com.opencsv.CSVReader)1 HeaderColumnNameMappingStrategy (com.opencsv.bean.HeaderColumnNameMappingStrategy)1 ComputeStatsRequest (com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest)1 ComputeStatsResponse (com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse)1 ComputeStats (com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats)1 EndpointConfigRequest (com.vmware.photon.controller.model.adapterapi.EndpointConfigRequest)1 AzureUriPaths (com.vmware.photon.controller.model.adapters.azure.AzureUriPaths)1 AzureCostConstants (com.vmware.photon.controller.model.adapters.azure.constants.AzureCostConstants)1 INTERNAL_REQUEST_TIMEOUT_SECONDS (com.vmware.photon.controller.model.adapters.azure.constants.AzureCostConstants.INTERNAL_REQUEST_TIMEOUT_SECONDS)1 AzureSubscriptionsEnumerationService (com.vmware.photon.controller.model.adapters.azure.ea.enumeration.AzureSubscriptionsEnumerationService)1 AzureSubscriptionsEnumerationRequest (com.vmware.photon.controller.model.adapters.azure.ea.enumeration.AzureSubscriptionsEnumerationService.AzureSubscriptionsEnumerationRequest)1 AzureCostHelper (com.vmware.photon.controller.model.adapters.azure.ea.utils.AzureCostHelper)1 AzureDetailedBillHandler (com.vmware.photon.controller.model.adapters.azure.ea.utils.AzureDetailedBillHandler)1 AzureResource (com.vmware.photon.controller.model.adapters.azure.model.cost.AzureResource)1 BillParsingStatus (com.vmware.photon.controller.model.adapters.azure.model.cost.BillParsingStatus)1