与大量数据嵌套的foreach导致内存不足exe123

Z时代
2024-01-10
分类：问答

我正在处理机器返回的数据。该机器通常用于大农场，它将采集不同深度的每个点的数据并使用不同的设备。与大量数据嵌套的foreach导致内存不足exe123

每场可容纳70-100字段和每个字段包含约10万条数据

在这里，我需要处理每一个数据和应用基于客户需求的一些计算。

我需要首先遍历农场，然后农场下的字段，然后每个农场使用的设备然后根据深度记录每个设备。所以最后我希望我做了大约10亿次的迭代。我的代码看起来像下面

public async void MigrateData() 
     { 
      masterData = await CatalogService.ExportMasterData(AppDataModel.Catalog, this.UserId); 
      foreach (var fieldItem in masterData.Fields) 
      { 
       var fieldReferenceId = fieldItem.ReferenceId; 
       dynamic loggedData = AppDataModel.Documents.LoggedData.Where(data => data.FieldId == fieldReferenceId); 
       dynamic fieldDbMappingData = fieldItem; 
       foreach (var data in loggedData) 
       { 
        yieldMaster.OperationalLogModalResponse = await YieldDataMigrationService.AddOperationalLogs("loggedDataDescription"); 
        yieldMaster.OperationalLogDataModelResponse = await YieldDataMigrationService.AddOperationalLogData(fieldDbMappingData, yieldMaster.OperationalLogModalResponse); 
        FetchContentData(data); 
       } 
      } 
     } 
private async void FetchContentData(LoggedData data) 
     { 
      foreach (var opdata in data.OperationData) 
      { 
       var filteredList = AggregateDataBasedOnFilter(opdata); 
       int count = filteredList.Count; 
       totalRecordCount += count; 
       for (int i = 1; i <= count; i = i + 1000) 
       { 
        var response = await YieldDataMigrationService.AddYiledData(filteredList.GetRange(i, i + 1000 >= count ? count - i : 1000)); 
       } 
      } 
      //Filter data based on timestamp values , get the first data in 5 seconds interval 
      System.GC.Collect(); 
     } 
private dynamic AggregateDataBasedOnFilter(OperationData opdata) 
     { 
      List<dynamic> listSpacialRecords = new List<dynamic>(); 
      IEnumerable<SpatialRecord> spacialRecords = opdata.GetSpatialRecords(); 
      spacialRecords = GetAggregateBasedOnTimeStamb(spacialRecords); 
      Nullable<Guid> productid; 
      for (int depth = 0; depth <= opdata.MaxDepth; depth++) 
      { 
       IEnumerable<DeviceElementUse> deviceElementUses = opdata.GetDeviceElementUses(depth); 
       StevProduct productDbMappingData = masterData.Products.Where(product => product.ReferenceId == opdata.ProductId).FirstOrDefault(); 
       if (productDbMappingData == null) 
       { 
        productid = null; 
       } 
       else 
       { 
        productid = productDbMappingData.Id; 
       } 
       foreach (var deviceElement in deviceElementUses) 
       { 
        List<dynamic> dvList = new List<dynamic>(); 
        IEnumerable<WorkingData> workingData = deviceElement.GetWorkingDatas(); 
        //foreach (var spacerecord in spacialRecords) 
        Parallel.ForEach(spacialRecords, (spacerecord) => 
        { 
         List<MeterValue> dat = new List<MeterValue>(); 
         var latitude = ((AgGateway.ADAPT.ApplicationDataModel.Shapes.Point)spacerecord.Geometry).Y; 
         var longitude = ((AgGateway.ADAPT.ApplicationDataModel.Shapes.Point)spacerecord.Geometry).X; 
         var timeStamp = spacerecord.Timestamp; 
         //Parallel.ForEach(workingData, (wdItem) => 
         foreach (var wdItem in workingData) 
         { 
          RepresentationValue spaceMeteredValue = spacerecord.GetMeterValue(wdItem); 
          if (spaceMeteredValue != null && wdItem.Representation != null) 
          { 
           //row[wdItem.Representation.Code] = meteredValue.Value.Value; 
           var objMeterValue = new MeterValue(); 
           objMeterValue.key = wdItem.Representation.Code; 
           objMeterValue.value = spaceMeteredValue.Designator != null ? Convert.ToString(spaceMeteredValue.Designator) : ""; 
           dat.Add(objMeterValue); 
          } 
         } 
         var newSpacialvalue = new 
         { 
          operationLogDataId = yieldMaster.OperationalLogDataModelResponse.Id, 
          order = deviceElement.Order, 
          totalDistanceTravelled = deviceElement.TotalDistanceTravelled, 
          totalElapsedTime = deviceElement.TotalElapsedTime, 
          uploadedOn = DateTime.Now.ToUniversalTime(), 
          collectedOn = timeStamp.ToUniversalTime(), 
          cropId = "8296e610-c055-11e7-851e-ad7650a5f99c", 
          productId = productid, 
          latitude = latitude, 
          longitude = longitude, 
          deviceConfigurationId = deviceElement.DeviceConfigurationId, 
          operationDataId = deviceElement.OperationDataId, 
          spatialRecords = dat, 
          depth = depth, 
          timeStamp = timeStamp, 
          totaldata = totalRecordCount 
         }; 
         lock (listSpacialRecords) 
         { 
          listSpacialRecords.Add(newSpacialvalue); 
         } 
        }); 
       } 
      } 
      listSpacialRecords = listSpacialRecords 
        .Skip(1) 
        .Aggregate(
         listSpacialRecords.Take(1).ToList(), 
         (a, x) => 
         { 
          if (x.timeStamp.Subtract(a.Last().timeStamp).TotalSeconds >= 10.0) 
          { 
           a.Add(x); 
          } 
          return a; 
         }); 
      GC.Collect(); 
      return listSpacialRecords; 
     }