MongoDB连接池优化 PengDi@2021/1/14

meter-reading-common/pom.xml

@@ -75,5 +75,13 @@
             <artifactId>spring-boot-starter-batch</artifactId>
             <version>2.2.5.RELEASE</version>
         </dependency>
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+        </dependency>
     </dependencies>
 </project>

meter-reading-common/src/main/java/com/huaxu/zoniot/config/MongoDBConfig.java

@@ -1,51 +1,232 @@
 package com.huaxu.zoniot.config;
 
-import com.mongodb.MongoClientOptions;
-import org.springframework.beans.factory.BeanFactory;
-import org.springframework.beans.factory.NoSuchBeanDefinitionException;
+import com.mongodb.*;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.data.convert.CustomConversions;
 import org.springframework.data.mongodb.MongoDbFactory;
-import org.springframework.data.mongodb.core.convert.DbRefResolver;
-import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver;
-import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
-import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
+import org.springframework.data.mongodb.core.convert.*;
 import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
+import org.springframework.validation.annotation.Validated;
 
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotEmpty;
+import java.util.ArrayList;
+import java.util.List;
 /**
  * @author pengdi
  */
 @Configuration
+@EnableConfigurationProperties(MongoDBConfig.MongoClientOptionProperties.class)
 public class MongoDBConfig {
     /**
-     * <p>解决 MongoSocketReadTimeoutException异常</p>
-     * @return
+     * 此Bean也是可以不显示定义的，如果我们没有显示定义生成MongoTemplate实例，
+     * SpringBoot利用我们配置好的MongoDbFactory在配置类中生成一个MongoTemplate，
+     * 之后我们就可以在项目代码中直接@Autowired了。因为用于生成MongoTemplate
+     * 的MongoDbFactory是我们自己在MongoConfig配置类中生成的，所以我们自定义的连接池参数也就生效了。
+     *
+     * @param mongoDbFactory mongo工厂
+     * @param converter      转换器
+     * @return MongoTemplate实例
      */
     @Bean
-    public MongoClientOptions mongoOptions() {
-        return MongoClientOptions
-            .builder()
-            .maxConnectionIdleTime(60000)
-            .build();
+    public MongoTemplate mongoTemplate(MongoDbFactory mongoDbFactory, MappingMongoConverter converter) {
+        MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory, converter);
+        // 设置读从库优先
+        mongoTemplate.setReadPreference(ReadPreference.secondaryPreferred());
+        return mongoTemplate;
     }
 
     /**
-     * <p>去除文档中的_class字段</p>
-     * @param factory
-     * @param context
-     * @param beanFactory
-     * @return
+     * 转换器
+     * MappingMongoConverter可以自定义mongo转换器，主要自定义存取mongo数据时的一些操作，例如 mappingConverter.setTypeMapper(new
+     * DefaultMongoTypeMapper(null)) 方法会将mongo数据中的_class字段去掉。
+     *
+     * @param factory     mongo工厂
+     * @param context     上下文
+     * @param conversions 自定义转换器
+     * @return 转换器对象
      */
     @Bean
-    public MappingMongoConverter mappingMongoConverter(MongoDbFactory factory, MongoMappingContext context, BeanFactory beanFactory) {
+    public MappingMongoConverter mappingMongoConverter(MongoDbFactory factory, MongoMappingContext context,
+                                                       MongoCustomConversions conversions) {
         DbRefResolver dbRefResolver = new DefaultDbRefResolver(factory);
         MappingMongoConverter mappingConverter = new MappingMongoConverter(dbRefResolver, context);
-        try {
-            mappingConverter.setCustomConversions(beanFactory.getBean(CustomConversions.class));
-        } catch (NoSuchBeanDefinitionException ignore) {
-        }
+        mappingConverter.setCustomConversions(conversions);
+        // remove _class field
         mappingConverter.setTypeMapper(new DefaultMongoTypeMapper(null));
         return mappingConverter;
     }
+
+    /**
+     * 自定义mongo连接池
+     *
+     * @param properties 属性配置类
+     * @return MongoDbFactory对象
+     */
+    @Bean
+    public MongoDbFactory mongoDbFactory(MongoClientOptionProperties properties) {
+
+        MongoClient mongoClient;
+
+        // 创建客户端参数
+        MongoClientOptions mongoClientOptions = mongoClientOptions(properties);
+
+        // 解析获取mongo服务地址
+        List<ServerAddress> serverAddressList = getServerAddress(properties.getAddress());
+
+        // 创建认证
+        MongoCredential mongoCredential = getCredential(properties);
+
+        // 创建客户端
+        if (null == mongoCredential) {
+            mongoClient = new MongoClient(serverAddressList, mongoClientOptions);
+        } else {
+            mongoClient = new MongoClient(serverAddressList, mongoCredential, mongoClientOptions);
+        }
+
+        return new SimpleMongoDbFactory(mongoClient, properties.getDatabase());
+    }
+
+    /**
+     * 创建认证
+     *
+     * @param properties 属性配置类
+     * @return 认证对象
+     */
+    private MongoCredential getCredential(MongoClientOptionProperties properties) {
+        if (!StringUtils.isEmpty(properties.getUsername()) && !StringUtils.isEmpty(properties.getPassword())) {
+            // 没有专用认证数据库则取当前数据库
+            String database = StringUtils.isEmpty(properties.getAuthenticationDatabase()) ?
+                    properties.getDatabase() : properties.getAuthenticationDatabase();
+            return MongoCredential.createCredential(properties.getUsername(), database,
+                    properties.getPassword().toCharArray());
+        }
+        return null;
+    }
+
+    /**
+     * 获取数据库服务地址
+     *
+     * @param mongoAddress 地址字符串
+     * @return 服务地址数组
+     */
+    private List<ServerAddress> getServerAddress(String mongoAddress) {
+        String[] mongoAddressArray = mongoAddress.trim().split(",");
+        List<ServerAddress> serverAddressList = new ArrayList<>(4);
+        for (String address : mongoAddressArray) {
+            String[] hostAndPort = address.split(":");
+            serverAddressList.add(new ServerAddress(hostAndPort[0], Integer.parseInt(hostAndPort[1])));
+        }
+        return serverAddressList;
+    }
+
+    /**
+     * mongo客户端参数配置
+     *
+     * @param properties 属性配置类
+     * @return mongo客户端参数配置对象
+     */
+    private MongoClientOptions mongoClientOptions(MongoClientOptionProperties properties) {
+        return MongoClientOptions.builder().applicationName(properties.getClientName()).
+                connectTimeout(properties.getConnectionTimeoutMs())
+                .maxConnectionIdleTime(properties.getMaxConnectionIdleTimeMs())
+                .maxConnectionLifeTime(properties.getMaxConnectionLifeTimeMs())
+                .socketTimeout(properties.getReadTimeoutMs())
+                .maxWaitTime(properties.getMaxWaitTimeMs())
+                .heartbeatFrequency(properties.getHeartbeatFrequencyMs())
+                .minHeartbeatFrequency(properties.getMinHeartbeatFrequencyMs())
+                .heartbeatConnectTimeout(properties.getHeartbeatConnectionTimeoutMs())
+                .heartbeatSocketTimeout(properties.getHeartbeatReadTimeoutMs())
+                .connectionsPerHost(properties.getConnectionsPerHost())
+                .minConnectionsPerHost(properties.getMinConnectionsPerHost())
+                .threadsAllowedToBlockForConnectionMultiplier(properties.getThreadsAllowedToBlockForConnectionMultiplier())
+                .readPreference(ReadPreference.secondaryPreferred())
+                .build();
+    }
+
+    @Getter
+    @Setter
+    @Validated
+    @ConfigurationProperties(prefix = "mongodb")
+    public static class MongoClientOptionProperties {
+
+        /**
+         * 基础连接参数
+         */
+        @NotEmpty
+        private String database; // 要连接的数据库
+        private String username; // 用户名
+        private String password; // 密码
+        @NotEmpty
+        private String address; // IP和端口（host:port），例如127.0.0.1:27017。集群模式用,分隔开，例如host1:port1,host2:port2
+        private String authenticationDatabase; // 设置认证数据库，如果有的话
+
+        /**
+         * 客户端连接池参数
+         */
+        @NotEmpty
+        private String clientName; // 客户端的标识，用于定位请求来源等，一般用程序名
+        @Min(value = 1)
+        private int connectionTimeoutMs; // TCP（socket）连接超时时间，毫秒
+        @Min(value = 1)
+        private int maxConnectionIdleTimeMs; // TCP（socket）连接闲置时间，毫秒
+        @Min(value = 1)
+        private int maxConnectionLifeTimeMs; // TCP（socket）连接最多可以使用多久，毫秒
+        @Min(value = 1)
+        private int readTimeoutMs; // TCP（socket）读取超时时间，毫秒
+        @Min(value = 1)
+        private int maxWaitTimeMs; // 当连接池无可用连接时客户端阻塞等待的最大时长，毫秒
+        @Min(value = 2000)
+        private int heartbeatFrequencyMs; // 心跳检测发送频率，毫秒
+        @Min(value = 300)
+        private int minHeartbeatFrequencyMs; // 最小的心跳检测发送频率，毫秒
+        @Min(value = 200)
+        private int heartbeatConnectionTimeoutMs; // 心跳检测连接超时时间，毫秒
+        @Min(value = 200)
+        private int heartbeatReadTimeoutMs; // 心跳检测读取超时时间，毫秒
+        @Min(value = 1)
+        private int connectionsPerHost; // 线程池允许的最大连接数
+        @Min(value = 1)
+        private int minConnectionsPerHost; // 线程池空闲时保持的最小连接数
+        @Min(value = 1)
+        // 计算允许多少个线程阻塞等待时的乘数，算法：threadsAllowedToBlockForConnectionMultiplier*maxConnectionsPerHost
+        private int threadsAllowedToBlockForConnectionMultiplier;
+    }
+//    /**
+//     * <p>解决 MongoSocketReadTimeoutException异常</p>
+//     * @return
+//     */
+//    @Bean
+//    public MongoClientOptions mongoOptions() {
+//        return MongoClientOptions
+//            .builder()
+//            .maxConnectionIdleTime(60000)
+//            .build();
+//    }
+//
+//    /**
+//     * <p>去除文档中的_class字段</p>
+//     * @param mongoDbFactory
+//     * @param context
+//     * @param beanFactory
+//     * @return
+//     */
+//    @Bean
+//    public MappingMongoConverter mappingMongoConverter(MongoDbFactory mongoDbFactory, MongoMappingContext context, BeanFactory beanFactory) {
+//        DbRefResolver dbRefResolver = new DefaultDbRefResolver(mongoDbFactory);
+//        MappingMongoConverter mappingConverter = new MappingMongoConverter(dbRefResolver, context);
+//        try {
+//            mappingConverter.setCustomConversions(beanFactory.getBean(CustomConversions.class));
+//        } catch (NoSuchBeanDefinitionException ignore) {
+//        }
+//        mappingConverter.setTypeMapper(new DefaultMongoTypeMapper(null));
+//        return mappingConverter;
+//    }
 }

meter-reading-common/src/main/java/com/huaxu/zoniot/service/impl/CommunityServiceImpl.java

@@ -43,7 +43,7 @@ public class CommunityServiceImpl implements CommunityService, ModelValidate<Com
         object.setCity(cityCode);
         object.setRegion(regionCode);
         validate(object);
-        Community community = communityMapper.findCommunityWithName(name, customerId , siteId, provinceCode, cityCode, regionCode);
+        Community community = communityMapper.findCommunityWithName(name, siteId  , customerId, provinceCode, cityCode, regionCode);
         return community;
     }
 

meter-reading-common/src/main/resources/application-common-dev.properties

@@ -38,7 +38,45 @@ spring.redis.lettuce.pool.max-idle=8
 spring.redis.lettuce.pool.min-idle=0
 spring.redis.lettuce.shutdown-timeout=100
 ######################################################MongoDB配置#####################################################
-spring.data.mongodb.uri=mongodb://114.135.61.188:17017/meter-reading-database
+#基础连接参数
+#要连接的数据库
+mongodb.database=meter-reading-database
+#用户名
+mongodb.username=
+#密码
+mongodb.password=
+#IP和端口（host:port），例如127.0.0.1:27017。集群模式用,分隔开，例如host1:port1,host2:port2
+mongodb.address=114.135.61.188:17017
+#设置认证数据库，如果有的话
+mongodb.authenticationDatabase=
+# 客户端连接池参数
+#客户端的标识，用于定位请求来源等，一般用程序名
+mongodb.clientName=${spring.application.name}
+#TCP（socket）连接超时时间，毫秒
+mongodb.connectionTimeoutMs=5000
+#TCP（socket）连接闲置时间，毫秒
+mongodb.maxConnectionIdleTimeMs=60000
+#TCP（socket）连接最多可以使用多久，毫秒
+mongodb.maxConnectionLifeTimeMs=300000
+#TCP（socket）读取超时时间，毫秒
+mongodb.readTimeoutMs=15000
+#当连接池无可用连接时客户端阻塞等待的最大时长，毫秒
+mongodb.maxWaitTimeMs=5000
+#心跳检测发送频率，毫秒
+mongodb.heartbeatFrequencyMs=20000
+#最小的心跳检测发送频率，毫秒
+mongodb.minHeartbeatFrequencyMs=8000
+#心跳检测连接超时时间，毫秒
+mongodb.heartbeatConnectionTimeoutMs=10000
+#心跳检测读取超时时间，毫秒
+mongodb.heartbeatReadTimeoutMs=15000
+#线程池允许的最大连接数
+mongodb.connectionsPerHost=20
+#线程池空闲时保持的最小连接数
+mongodb.minConnectionsPerHost=4
+#计算允许多少个线程阻塞等待时的乘数，算法：threadsAllowedToBlockForConnectionMultiplier*maxConnectionsPerHost
+mongodb.threadsAllowedToBlockForConnectionMultiplier=10
+#spring.data.mongodb.uri=mongodb://114.135.61.188:17017/meter-reading-database
 logging.level.org.springframework.data.mongodb.core=DEBUG
 ######################################################SpringBatch配置#####################################################
 # 默认情况下，项目启动时就会自动执行配置好的批处理操作。这里将其设为不自动执行，后面我们通过手动触发执行批处理

meter-reading-common/src/main/resources/application-common-sit.properties

@@ -38,7 +38,45 @@ spring.redis.lettuce.pool.max-idle=8
 spring.redis.lettuce.pool.min-idle=0
 spring.redis.lettuce.shutdown-timeout=100
 ######################################################MongoDB配置#####################################################
-spring.data.mongodb.uri=mongodb://114.135.61.188:17017/meter-reading-database
+#基础连接参数
+#要连接的数据库
+mongodb.database=meter-reading-database
+#用户名
+mongodb.username=
+#密码
+mongodb.password=
+#IP和端口（host:port），例如127.0.0.1:27017。集群模式用,分隔开，例如host1:port1,host2:port2
+mongodb.address=114.135.61.188:17017
+#设置认证数据库，如果有的话
+mongodb.authenticationDatabase=
+# 客户端连接池参数
+#客户端的标识，用于定位请求来源等，一般用程序名
+mongodb.clientName=${spring.application.name}
+#TCP（socket）连接超时时间，毫秒
+mongodb.connectionTimeoutMs=5000
+#TCP（socket）连接闲置时间，毫秒
+mongodb.maxConnectionIdleTimeMs=60000
+#TCP（socket）连接最多可以使用多久，毫秒
+mongodb.maxConnectionLifeTimeMs=300000
+#TCP（socket）读取超时时间，毫秒
+mongodb.readTimeoutMs=15000
+#当连接池无可用连接时客户端阻塞等待的最大时长，毫秒
+mongodb.maxWaitTimeMs=5000
+#心跳检测发送频率，毫秒
+mongodb.heartbeatFrequencyMs=20000
+#最小的心跳检测发送频率，毫秒
+mongodb.minHeartbeatFrequencyMs=8000
+#心跳检测连接超时时间，毫秒
+mongodb.heartbeatConnectionTimeoutMs=10000
+#心跳检测读取超时时间，毫秒
+mongodb.heartbeatReadTimeoutMs=15000
+#线程池允许的最大连接数
+mongodb.connectionsPerHost=20
+#线程池空闲时保持的最小连接数
+mongodb.minConnectionsPerHost=4
+#计算允许多少个线程阻塞等待时的乘数，算法：threadsAllowedToBlockForConnectionMultiplier*maxConnectionsPerHost
+mongodb.threadsAllowedToBlockForConnectionMultiplier=10
+#spring.data.mongodb.uri=mongodb://114.135.61.188:17017/meter-reading-database
 logging.level.org.springframework.data.mongodb.core=DEBUG
 ######################################################SpringBatch配置#####################################################
 # 默认情况下，项目启动时就会自动执行配置好的批处理操作。这里将其设为不自动执行，后面我们通过手动触发执行批处理

meter-reading-tianjin/src/main/java/com/huaxu/zoniot/MeterReadingTianJinApplication.java

@@ -2,6 +2,7 @@ package com.huaxu.zoniot;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableAsync;
 
 /**
  * <p></p>
@@ -10,6 +11,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
  * @Date 2020/12/16 21:09
  * @Version 1.0
  */
+@EnableAsync
 @SpringBootApplication
 public class MeterReadingTianJinApplication {
 

meter-reading-tianjin/src/main/resources/application-tianjin-dev.properties

@@ -2,6 +2,7 @@ server.port=8001
 logging.level.root=info
 logging.file.path=./logs
 server.servlet.context-path=/readmeter
+spring.application.name=meter-reading-tianjin
 ##############################################ÒµÎñÅäÖÃ##########################################
 api.key=hauxureadmeter
 netcode=20191118
@@ -10,4 +11,4 @@ province.code=120000
 city.code=120100
 region.code=120112
 customer.code=54
-device.type.code=58
+device.type.code=20