通用推送数据功能优化@2020/11/17 by pengdi

smart-city-platform/src/main/java/com/bz/smart_city/dao/DeviceDataDimMapper.java

@@ -57,7 +57,14 @@ public interface DeviceDataDimMapper {
     		@Param("offset") Integer offset,
     		@Param("limit") Integer limit
     		);
-    
+
+    List<Map<String, String>> queryDeviceDataWithDateRange(
+            @Param("deviceId") Long  deviceId ,
+            @Param("deviceTypeId") Integer  deviceTypeId ,
+            @Param("beginDay") Integer beginDay ,
+            @Param("endDay")   Integer endDay
+            );
+
     List<Map<String,Object>> getDeviceDataByTime(@Param("deviceId") Long deviceId , @Param("sendTime") Date sendTime,@Param("deviceTypeId") Integer deviceTypeId);
 
 }

smart-city-platform/src/main/java/com/bz/smart_city/dao/DeviceMapper.java

@@ -134,7 +134,11 @@ public interface DeviceMapper {
             @Param("buildings") String buildings,
             @Param("beginDate") Date beginDate,
             @Param("endDate") Date endDate);
-
+    List<DeviceDto> getDeviceListByRange(
+            @Param("customers") String customers,
+            @Param("communitys") String communitys,
+            @Param("buildings") String buildings
+    );
     List<PlanCommunityDTO> findFaultByCommunity(
             @Param("customerId") Integer customerId,
             @Param("communityName") String communityName);

smart-city-platform/src/main/java/com/bz/smart_city/entity/DeviceDataDim.java

@@ -64,6 +64,8 @@ public class DeviceDataDim {
     private String equipmentType; 
     private Integer startDate ; 
     private Integer endDate ; 
-    
     private String meterCode;
+
+    @ApiModelProperty(value="数据描述")
+    private String measuringDataDesc ;
 }

smart-city-platform/src/main/java/com/bz/smart_city/service/impl/DeviceDataPushServiceImpl.java

@@ -5,15 +5,11 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.text.SimpleDateFormat;
 import java.time.LocalDateTime;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -78,28 +74,149 @@ public class DeviceDataPushServiceImpl implements DeviceDataPushService , Initia
 	
 	@Autowired
 	JobAndTriggerService jobAndTriggerService;
-	
 	@Override
 	public void generalPushMethod(DeviceDataPushConfig config) {
 		log.info("begin generalPushMethod ,config = {}",JSON.toJSONString(config));
-		// 1,查询推送记录 
-		PushLog lastLog = pushLogMapper.findLastRecord(config.getId(), 2); // 1档案；2数据
+		// 1,计算推送数据的起始时间
+		Date beginDate = calcBeginDate(config) ;
+		Date endDate = calcEndDate();
+		Integer endDay = Integer.parseInt(DateTimeUtil.formatDate(endDate,"yyyyMMdd"));
+		Integer beginDay = Integer.parseInt(DateTimeUtil.formatDate(beginDate,"yyyyMMdd"));
+		// 2,获取所需要的设备列表
+		List<DeviceDto> deviceList = getPushMeterList(config);
+		// 3,根据设备列表查询时间段范围内的数据,并进行分页推送
+		Integer maxLimit = config.getMaxLimit();
+		Integer total = 0 ;
+		List<Map<String,Object>> dataMapList = new ArrayList<>();
+		for(DeviceDto deviceDto : deviceList){
+            List<Map<String, String>> dataItemList = queryDeviceDataByRange(deviceDto, endDay, beginDay);
+            for(Map<String, String> dataItemMap :  dataItemList){
+                total++ ;
+                Map<String, Object> dataMap = buildDeviceDataMap(deviceDto, dataItemMap);
+                dataMapList.add(dataMap);
+                if(dataMapList.size()  == maxLimit){
+                    // 触发批量发送
+                    sendBatchData( dataMapList , beginDate ,endDate , config);
+                    // 清空数组
+                    dataMapList.clear();
+                }
+            }
+        }
+		if(dataMapList.size() != 0){
+			sendBatchData( dataMapList , beginDate ,endDate , config);
+		}
+		log.info("end generalPushMethod ,Totol = {}!",total);
+	}
+
+
+
+	protected Map<String,Object> buildDeviceDataMap(DeviceDto device , Map<String, String> dataItem){
+		Map<String,Object> dataMap = new HashMap<String,Object>();
+		dataMap.put("deviceNo", device.getDeviceNo());
+		dataMap.put("deviceType", device.getEquipmentType());
+		dataMap.put("sendTime", dataItem.get("sendTime"));
+		dataMap.put("datas", buildDeviceData(dataItem.get("deviceData")));
+		return dataMap ;
+	}
+
+	protected List<Map<String, String>> buildDeviceData(String dataString){
+		List<Map<String, String>> dataList = new ArrayList<>();
+		// 使用逗号分隔得到多个测点的数据
+		String[] splits = dataString.split(",");
+		for(String split : splits){
+			// 使用|分隔将行转为列
+			String[] measuringData = split.split("\\|");
+			Map<String, String> measuringDataMap = new HashMap<>(measuringData.length);
+			measuringDataMap.put("measuringCode",measuringData[0]);
+			measuringDataMap.put("measuringName",measuringData[1]);
+			measuringDataMap.put("measuringData",measuringData[2]);
+			measuringDataMap.put("measuringUnit",measuringData[3]);
+			measuringDataMap.put("measuringDataDesc",measuringData[4]);
+			dataList.add(measuringDataMap);
+		}
+		return dataList ;
+	}
+	protected List<Map<String,String>> queryDeviceDataByRange(DeviceDto device,Integer endDay ,Integer beginDay){
+		List<Map<String, String>> dataItemList = deviceDataDimMapper.queryDeviceDataWithDateRange(device.getId(),
+				device.getDeviceType(),
+				beginDay,
+				endDay);
+		return dataItemList ;
+	}
+
+	protected List<DeviceDto> getPushMeterList(DeviceDataPushConfig config){
+		String pushCustomers = config.getPushCustomers();
+		String pushCommunities = config.getPushCommunitys();
+		String pushBuildings = config.getPushBuildings();
+		List<DeviceDto> deviceList = deviceMapper.getDeviceListByRange(pushCustomers, pushCommunities, pushBuildings);
+		return deviceList ;
+	}
+
+	/**
+	 * <p>
+	 *     从推送日志中获取开始时间,如果有推送日志则取上一次推送时间的后一天的开始时间
+	 * 	   如果没有推送日志则根据是否需要历史数据来取
+	 * 	   	   如果不需要历史数据,则取前一天的开始时间
+	 * 	   	   如果需要历史数据,则取1900-01-01
+	 * </p>
+	 * @param config
+	 * @return
+	 */
+	protected Date calcBeginDate(DeviceDataPushConfig config){
+		// 1,查询推送记录 // 1档案；2数据
+		PushLog lastLog = pushLogMapper.findLastRecord(config.getId(), 2);
 		// 2,计算推送数据时间；如果为首次推送，则判断是否需要将历史数据推送给对方；
 		Date beginDate = null ;
-		Date endDate = new Date();
 		if(lastLog != null) {
-			beginDate = lastLog.getPushTime() ; 
+			beginDate = getDateStartTime(DateTimeUtil.after(lastLog.getPushTime(),1));
 		}
 		else {
-			Integer isNeedHistory = config.getIsNeedHistory(); // 是否需要历史数据  1是  ； 0否
+			// 是否需要历史数据  1是  ； 0否
+			Integer isNeedHistory = config.getIsNeedHistory();
 			if(isNeedHistory == 1) {
 				beginDate = DateTimeUtil.parseDate("1900-01-01", "yyyy-MM-dd");
 			}
 			else {
-				long interval = CronUtil.calcInterval(config.getPeriod());
-				beginDate = new Date(endDate.getTime() - interval);
+				beginDate = getDateStartTime(DateTimeUtil.beforeNow(1));
 			}
 		}
+		return beginDate ;
+	}
+	/**
+	 * 计算推送的截止时间,默认是今日推送前一天的数据
+	 * @return
+	 */
+	protected  Date calcEndDate(){
+		Date date = DateTimeUtil.beforeNow(1);
+		return getDateEndTime(date) ;
+	}
+
+	protected  Date getDateStartTime(Date date){
+		Calendar calendar = new GregorianCalendar();
+		calendar.setTime(date);
+		calendar.set(Calendar.HOUR_OF_DAY,0);
+		calendar.set(Calendar.MINUTE,0);
+		calendar.set(Calendar.SECOND,0);
+		calendar.set(Calendar.MILLISECOND,0);
+		Date dayStart = calendar.getTime();
+		return dayStart ;
+	}
+
+	protected Date getDateEndTime(Date date){
+		Calendar calendar = new GregorianCalendar();
+		calendar.setTime(date);
+		calendar.set(Calendar.HOUR_OF_DAY,23);
+		calendar.set(Calendar.MINUTE,59);
+		calendar.set(Calendar.SECOND,59);
+		calendar.set(Calendar.MILLISECOND,999);
+		Date dayEnd = calendar.getTime();
+		return dayEnd;
+	}
+	public void generalPushMethodTmp(DeviceDataPushConfig config) {
+		// 1,计算推送数据的起始时间
+		Date beginDate = calcBeginDate(config) ;
+		Date endDate = new Date();
+
 		// 3,根据推送数据时间进行数据查询
 		String pushCustomers = config.getPushCustomers();
 		String pushCommunitys = config.getPushCommunitys();
@@ -171,7 +288,8 @@ public class DeviceDataPushServiceImpl implements DeviceDataPushService , Initia
 			pushLog.setPushStatus(isSuccess ? 1 : 0);
 		}catch (Exception e) {
 			log.error("推送失败", e);
-			pushLog.setPushStatus(0); //推送失败
+			//推送失败
+			pushLog.setPushStatus(0);
 			pushLog.setRemark(e.getMessage().length() > 512 ? e.getMessage().substring(0, 512) : e.getMessage());
 		}finally {
 			// 3,保存推送日志
@@ -337,7 +455,6 @@ public class DeviceDataPushServiceImpl implements DeviceDataPushService , Initia
 	
 	/**
 	  * 获取消息签名
-	 * @param msg
 	 * @return
 	 */
 	public static String getSign(PostData data) {

smart-city-platform/src/main/resources/mapper/DeviceDataDimMapper.xml

@@ -281,6 +281,51 @@
 		 		</trim>
 			) 
     </select>
+    <select id="queryDeviceDataWithDateRange" resultType="java.util.HashMap">
+        SELECT
+        DATE_FORMAT(t1.send_time,'%Y-%m-%d %H:%i:%s') as sendTime,
+        GROUP_CONCAT(CONCAT_WS('|',
+            t1.measuring_code,
+            t1.measuring_name,
+            t1.measuring_data,
+            IFNULL(t1.measuring_unit,' '),
+            IFNULL(t1.data_desc,' '))) as deviceData
+        FROM
+        (
+            SELECT
+            dd.send_date,
+            dd.send_time,
+            dd.measuring_code,
+            md.measuring_name ,
+            measuring_data,
+            dd.measuring_unit,
+            tmp.data_name as data_desc
+            FROM
+            sc_device_data_dim dd
+            LEFT JOIN sc_tpl_measuring_desc md ON ( dd.measuring_id = md.measuring_id AND md.device_type = #{deviceTypeId} )
+            LEFT JOIN (
+                SELECT
+                    device_type,
+                    measuring_id,
+                    measuring_code,
+                    measuring_name,
+                    measuring_unit,
+                    data_value,
+                    data_name
+                FROM
+                    sc_tpl_measuring_desc md
+                LEFT JOIN sc_tpl_measuring_data_def mdd ON ( md.id = mdd.measuring_desc )
+                WHERE
+                md.device_type = #{deviceTypeId}
+            ) tmp ON ( dd.measuring_id = tmp.measuring_id AND dd.measuring_data = tmp.data_value )
+            WHERE
+            device_id = #{deviceId,jdbcType=BIGINT}
+            and send_date >= #{beginDay}
+            and send_date <![CDATA[ <= ]]> #{endDay}
+        ) t1
+        group by
+        t1.send_time
+    </select>
     <select id="queryDataWithPage" resultType="com.bz.smart_city.entity.DeviceDataDim">
     	SELECT
 			distinct ddd.device_no,ddd.device_id,d.device_type,dt.equipment_type,ddd.send_time

smart-city-platform/src/main/resources/mapper/DeviceMapper.xml

@@ -959,6 +959,35 @@
 		d.`status` = 1 
 		AND d.customer_id IN ( 3, 4, 5, 6 )
 	</select>
+    <select id="getDeviceListByRange" resultType="com.bz.smart_city.dto.DeviceDto">
+        SELECT
+            d.id ,
+            d.device_no  ,
+            dt.equipment_type,
+            d.device_type
+        FROM
+        sc_device d
+        LEFT JOIN sc_device_type dt ON ( d.device_type = dt.id )
+        LEFT JOIN sc_building b ON ( d.building_id = b.id )
+        LEFT JOIN sc_community c ON ( b.community = c.id )
+        WHERE
+        d.`status` = 1
+        AND d.sys_id != - 99
+        AND d.sys_id != 55
+        AND d.id IN ( SELECT dd.device_id FROM sc_device_dimension dd WHERE
+        <trim prefixOverrides="or" >
+            <if test="customers != null and customers != '' ">
+                FIND_IN_SET( dd.customer, #{customers} )
+            </if>
+            <if test="communitys != null and communitys != '' ">
+                or FIND_IN_SET( dd.community, #{communitys} )
+            </if>
+            <if test="buildings != null and buildings != '' ">
+                or FIND_IN_SET( dd.building, #{buildings} )
+            </if>
+        </trim>
+        )
+    </select>
     <select id="getPushDeviceDataList" resultType="java.util.HashMap">
         SELECT
         d.device_no AS deviceNo,

smart-city-platform/src/test/java/com/bz/smart_city/DeviceDataPushServiceTest.java

@@ -1,5 +1,7 @@
 package com.bz.smart_city;
 
+import com.bz.smart_city.dao.DeviceDataPushConfigMapper;
+import com.bz.smart_city.entity.DeviceDataPushConfig;
 import com.bz.smart_city.service.impl.ZunYiNewRegionSyncServiceImpl;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -34,6 +36,10 @@ public class DeviceDataPushServiceTest {
 	@Autowired
 	JobAndTriggerService jobAndTriggerService;
 
+	@Autowired
+	DeviceDataPushConfigMapper deviceDataPushConfigMapper ;
+
+
 
 	@Test
 	public void contextLoads() {
@@ -45,6 +51,13 @@ public class DeviceDataPushServiceTest {
 		entity.setJobGroup("设备档案推送");
 		jobAndTriggerService.triggerJob(entity);
 	}
+
+	@Test
+	public void deviceDataPushJobTest2(){
+		int configId = 7;
+		DeviceDataPushConfig config = deviceDataPushConfigMapper.findConfigById(configId);
+		deviceDataPushService.generalPushMethod(config);
+	}
 	@Test
 	public void deviceDataPushJobTest() {
 		QuartzEntity entity = new QuartzEntity();