处理线程优化，减轻jvm压力

src/main/java/com/skyversation/poiaddr/addquery/AddressQueryEngine.java

@@ -54,9 +54,9 @@ public class AddressQueryEngine {
             }
             // 优先进行数据库查询
             List<AmapAddressV3> dbPois = AreaService.getInstance().getAddressPoisByAddr(addr);
-            JSONArray pois = new JSONArray();
             if (dbPois != null && dbPois.size() > 0) {
-                System.out.println("数据库查询有结果！！！！");
+//                System.out.println("数据库查询有结果！！！！");
+                JSONArray pois = new JSONArray();
                 pois.addAll(dbPois);
                 return new TransfromDataTool().gdV3ResultToResult(addr, pois, false);
             }

src/main/java/com/skyversation/poiaddr/service/impl/YyskAddressStandardizationServiceImpl.java

@@ -11,6 +11,7 @@ import com.skyversation.poiaddr.util.AddressMatcher;
 import com.skyversation.poiaddr.util.AddressTools;
 import com.skyversation.poiaddr.util.Coordinate;
 import com.skyversation.poiaddr.util.ExcelReaderUtils;
+import com.skyversation.poiaddr.util.tasks.ScheduledTasks;
 import org.springframework.stereotype.Service;
 
 import java.sql.PreparedStatement;
@@ -311,31 +312,17 @@ public class YyskAddressStandardizationServiceImpl {
     public void searchLoadFileData(String fileRootPath, Integer startFileIndex, Integer endFileIndex, Integer sort) {
         // 任务总数
         int totalTasks = Math.abs(endFileIndex - startFileIndex);
-        // 创建一个最大线程数一半的线程池
-        ExecutorService executorService = Executors.newFixedThreadPool(6);
-        List<Future<?>> futures = new ArrayList<>();
         // 提交任务到线程池
         for (int i = 0; i < totalTasks; i++) {
+            long startTime = System.currentTimeMillis();
             final int taskId = sort * (i + 1) + startFileIndex;
-            futures.add(executorService.submit(() -> {
-                // 任务执行
-                System.out.println("<<<<<<------Executing task: " + taskId + " on thread: " + Thread.currentThread().getName() + "------>>>>>>>>");
-                startMutilExecutor(taskId, fileRootPath + "datas\\yysk_dmdz_address_standardization_200000_" + taskId + ".xlsx",
-                        fileRootPath + "output\\yysk_dmdz_address_standardization_200000_result_" + taskId + ".xlsx");
-            }));
-        }
-
-        for (Future<?> future : futures) {
-            try {
-                future.get();
-            } catch (InterruptedException | ExecutionException e) {
-                System.err.println("线程异常：" + e);
-            }
+            // 任务执行
+            startMutilExecutor(taskId, fileRootPath + "datas\\yysk_dmdz_address_standardization_200000_" + taskId + ".xlsx",
+                    fileRootPath + "output\\yysk_dmdz_address_standardization_200000_result_" + taskId + ".xlsx");
+            long endTime = System.currentTimeMillis();
+            ScheduledTasks.taskRunStatus = false;
+            System.out.println("单个文件处理完成！用时" + (endTime - startTime) / 1000 + "秒！");
         }
-        // 关闭线程池
-        executorService.shutdown();
-        System.out.println("<<<<<--- 任务完成！！！----->>>>>>>");
-
     }
 
     public void startMutilExecutor(int startFileIndex, String filePath, String outPath) {
@@ -356,10 +343,8 @@ public class YyskAddressStandardizationServiceImpl {
         if (listData != null && listData.size() > 0) {
 //              批量更新处理后的数据
             List<YyskAddressStandardization> resultDataList = runExecutorService(listData);
-            listData.clear();
             if (resultDataList.size() > 0) {
                 ExcelReaderUtils.convertYYListToMapList(resultDataList, outPath);
-                resultDataList.clear();
             }
             long endTime = System.currentTimeMillis();
             System.out.println("处理单批次用时" + (endTime - startTime) / 1000 + "秒！");
@@ -370,120 +355,121 @@ public class YyskAddressStandardizationServiceImpl {
 
     public List<YyskAddressStandardization> runExecutorService(List<YyskAddressStandardization> listData) {
         // 创建线程池
-//        ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
-//        List<Future<?>> futures = new ArrayList<>();
-        int index = 1;
-        int total = listData.size();
+        ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+        List<Future<?>> futures = new ArrayList<>();
+//        int index = 1;
+//        int total = listData.size();
         for (YyskAddressStandardization item : listData) {
-            System.out.println("<<<---- 开始查询第： " + index + " 条数据。。。共：" + total + "条数据--->>>>>>>");
-            index++;
-//            futures.add(executorService.submit(() -> {
-            List<String> addrList = new ArrayList<>();
-            String[] addr = AddressMatcher.matchProvinceAndCity(item.getAddress());
-            if (addr != null && !addr[0].contains("上海")) {
+//            index++;
+//            int finalIndex = index;
+            futures.add(executorService.submit(() -> {
+//                System.out.println("<<<---- 开始查询第： " + finalIndex + " 条数据。。。共：" + total + "条数据--->>>>>>>");
+                List<String> addrList = new ArrayList<>();
+                String[] addr = AddressMatcher.matchProvinceAndCity(item.getAddress());
+                if (addr != null && !addr[0].contains("上海")) {
 //                  匹配全国省名
-                item.setProvinces(addr[0]);
+                    item.setProvinces(addr[0]);
 //                  遍历全国城市名称匹配搜索地址市名
-                item.setMarket(addr[1]);
-                item.setMatchLevel("rule_0");
-            } else {
-                addrList.add(item.getAddress());
-                String[] strs = AddressTools.parseAddressJZ(item.getAddress());
-                if (strs != null) {
-                    item.setProvinces(strs[0] == null ? "" : strs[0]);
-                    item.setMarket(strs[0] == null ? "" : strs[0]);
-                    item.setDistinguish(strs[1] == null ? "" : strs[1]);
-                    item.setStreetTown(strs[2] == null ? "" : strs[2]);
-                }
-            }
-            if (addrList.size() > 0) {
-                //  TODO 开始查询
-                AddressResult addressResult = AddressQueryEngine.getInstance().commonSearchByName(addrList);
-                System.out.println("<<<---- 第： " + index + " 条数据返回数据，数据为：！！！" + addressResult.getMessage() + "   ----->>>>>");
-                if (addressResult.getData() == null || addressResult.getData().size() < 1) {
-                    item.setMatchLevel("异常");
-                    item.setReturnAddress("未匹配到符合规则的结果");
+                    item.setMarket(addr[1]);
+                    item.setMatchLevel("rule_0");
                 } else {
-                    try {
-                        for (AddressResult.ContentBean contentBean : addressResult.getData()) {
-                            String resultAddrKey = contentBean.getAddress();
-                            if (resultAddrKey != null && contentBean.getLon() != null && contentBean.getLat() != null && AddressTools.isOtherDistrictThanSongJiang2(resultAddrKey)) {
+                    addrList.add(item.getAddress());
+                    String[] strs = AddressTools.parseAddressJZ(item.getAddress());
+                    if (strs != null) {
+                        item.setProvinces(strs[0] == null ? "" : strs[0]);
+                        item.setMarket(strs[0] == null ? "" : strs[0]);
+                        item.setDistinguish(strs[1] == null ? "" : strs[1]);
+                        item.setStreetTown(strs[2] == null ? "" : strs[2]);
+                    }
+                }
+                if (addrList.size() > 0) {
+                    //  TODO 开始查询
+                    AddressResult addressResult = AddressQueryEngine.getInstance().commonSearchByName(addrList);
+//                    System.out.println("<<<---- 第： " + finalIndex + " 条数据返回数据，数据为：！！！" + addressResult.getMessage() + "   ----->>>>>");
+                    if (addressResult.getData() == null || addressResult.getData().size() < 1) {
+                        item.setMatchLevel("异常");
+                        item.setReturnAddress("未匹配到符合规则的结果");
+                    } else {
+                        try {
+                            for (AddressResult.ContentBean contentBean : addressResult.getData()) {
+                                String resultAddrKey = contentBean.getAddress();
+                                if (resultAddrKey != null && contentBean.getLon() != null && contentBean.getLat() != null && AddressTools.isOtherDistrictThanSongJiang2(resultAddrKey)) {
 //                                  去除特殊字符
-                                resultAddrKey = AddressTools.getInstance().deleteStr(contentBean.getAddress());
+                                    resultAddrKey = AddressTools.getInstance().deleteStr(contentBean.getAddress());
 //                                  遍历全国省份名称得到省名
-                                item.setProvinces(AddressTools.isOtherDistrictThanShangHai(resultAddrKey));
+                                    item.setProvinces(AddressTools.isOtherDistrictThanShangHai(resultAddrKey));
 //                                  写入街镇数据
-                                if (contentBean.getAdname() != null && !contentBean.getAdname().isEmpty()) {
-                                    item.setStreetTown(contentBean.getAdname());
-                                } else if (contentBean.getTownJson() != null && contentBean.getTownJson().getString("name") != null) {
-                                    item.setStreetTown(contentBean.getTownJson().getString("name"));
-                                } else {
-                                    item.setStreetTown("");
-                                }
+                                    if (contentBean.getAdname() != null && !contentBean.getAdname().isEmpty()) {
+                                        item.setStreetTown(contentBean.getAdname());
+                                    } else if (contentBean.getTownJson() != null && contentBean.getTownJson().getString("name") != null) {
+                                        item.setStreetTown(contentBean.getTownJson().getString("name"));
+                                    } else {
+                                        item.setStreetTown("");
+                                    }
 //                                  写入上海市市名
-                                item.setMarket(AddressTools.isOtherDistrictThanShangHai(resultAddrKey));
+                                    item.setMarket(AddressTools.isOtherDistrictThanShangHai(resultAddrKey));
 //                                  写入区名
-                                if (contentBean.getCityname() != null && !contentBean.getCityname().isEmpty()) {
-                                    item.setDistinguish(contentBean.getCityname());
-                                } else if (contentBean.getAdJson() != null && contentBean.getAdJson().getString("name") != null) {
-                                    item.setDistinguish(contentBean.getAdJson().getString("name"));
-                                } else {
-                                    item.setDistinguish("");
-                                }
+                                    if (contentBean.getCityname() != null && !contentBean.getCityname().isEmpty()) {
+                                        item.setDistinguish(contentBean.getCityname());
+                                    } else if (contentBean.getAdJson() != null && contentBean.getAdJson().getString("name") != null) {
+                                        item.setDistinguish(contentBean.getAdJson().getString("name"));
+                                    } else {
+                                        item.setDistinguish("");
+                                    }
 //                                  写入sh2000坐标系
-                                if (contentBean.getLocation() != null && !contentBean.getLocation().trim().isEmpty()) {
-                                    String[] latLonStr = contentBean.getLocation().split(",");
-                                    item.setLat(latLonStr[0]);
-                                    item.setLon(latLonStr[1]);
-                                }
-                                item.setReturnAddress(resultAddrKey);
-                                item.setMatchLevel(contentBean.getScore());
+                                    if (contentBean.getLocation() != null && !contentBean.getLocation().trim().isEmpty()) {
+                                        String[] latLonStr = contentBean.getLocation().split(",");
+                                        item.setLat(latLonStr[0]);
+                                        item.setLon(latLonStr[1]);
+                                    }
+                                    item.setReturnAddress(resultAddrKey);
+                                    item.setMatchLevel(contentBean.getScore());
 //                                  判断并写入居委
-                                if (contentBean.getCommunity() != null && !contentBean.getCommunity().isEmpty()) {
-                                    item.setResidentialCommittee(contentBean.getCommunity());
-                                } else if (contentBean.getCjJson() != null && contentBean.getCjJson().containsKey("居委_1")) {
-                                    item.setResidentialCommittee(contentBean.getCjJson().getString("居委_1"));
+                                    if (contentBean.getCommunity() != null && !contentBean.getCommunity().isEmpty()) {
+                                        item.setResidentialCommittee(contentBean.getCommunity());
+                                    } else if (contentBean.getCjJson() != null && contentBean.getCjJson().containsKey("居委_1")) {
+                                        item.setResidentialCommittee(contentBean.getCjJson().getString("居委_1"));
+                                    } else {
+                                        item.setResidentialCommittee("");
+                                    }
+                                    String oldAddress = "";
+                                    try {
+                                        oldAddress = Objects.requireNonNull(AddressTools.parseAddressJZ(contentBean.getSearchAddress()))[3].
+                                                replaceAll(item.getMarket().equals("[]") ? "" : item.getMarket(), "").
+                                                replaceAll(item.getDistinguish().equals("[]") ? "" : item.getDistinguish(), "").
+                                                replaceAll(item.getStreetTown().equals("[]") ? "" : item.getStreetTown(), "").
+                                                replaceAll(item.getResidentialCommittee().equals("[]") ? "" : item.getResidentialCommittee(), "");
+                                    } catch (Exception e) {
+                                        e.printStackTrace();
+                                    }
+                                    item.setStandardAddress(item.getMarket() + item.getDistinguish() + item.getStreetTown() + AddressQueryEngine.townReplace_ct(oldAddress));
+                                    break;
                                 } else {
-                                    item.setResidentialCommittee("");
-                                }
-                                String oldAddress = "";
-                                try {
-                                    oldAddress = Objects.requireNonNull(AddressTools.parseAddressJZ(contentBean.getSearchAddress()))[3].
-                                            replaceAll(item.getMarket().equals("[]") ? "" : item.getMarket(), "").
-                                            replaceAll(item.getDistinguish().equals("[]") ? "" : item.getDistinguish(), "").
-                                            replaceAll(item.getStreetTown().equals("[]") ? "" : item.getStreetTown(), "").
-                                            replaceAll(item.getResidentialCommittee().equals("[]") ? "" : item.getResidentialCommittee(), "");
-                                } catch (Exception e) {
-                                    e.printStackTrace();
+                                    item.setMatchLevel("异常");
+                                    item.setReturnAddress("结果处理异常");
                                 }
-                                item.setStandardAddress(item.getMarket() + item.getDistinguish() + item.getStreetTown() + AddressQueryEngine.townReplace_ct(oldAddress));
-                                break;
-                            } else {
-                                item.setMatchLevel("异常");
-                                item.setReturnAddress("结果处理异常");
                             }
+                        } catch (Exception e) {
+                            item.setMatchLevel("异常");
+                            e.printStackTrace();
+                            System.err.println("查询结果处理异常：" + e);
                         }
-                    } catch (Exception e) {
-                        item.setMatchLevel("异常");
-                        e.printStackTrace();
-                        System.err.println("查询结果处理异常：" + e);
                     }
                 }
-            }
-            item.setUpdateTime(new Date());
-//            }));
+                item.setUpdateTime(new Date());
+            }));
         }
-
         // 等待所有任务完成
-//        for (Future<?> future : futures) {
-//            try {
-//                future.get();
-//            } catch (InterruptedException | ExecutionException e) {
-//                System.err.println("线程异常：" + e);
-//            }
-//        }
-//        // 关闭线程池
-//        executorService.shutdown();
+        for (Future<?> future : futures) {
+            try {
+                future.get();
+            } catch (InterruptedException | ExecutionException e) {
+                e.printStackTrace();
+                System.err.println("线程异常：" + e);
+            }
+        }
+        // 关闭线程池
+        executorService.shutdown();
         return listData;
     }
 }

src/main/java/com/skyversation/poiaddr/util/ExcelReaderUtils.java

@@ -110,6 +110,7 @@ public class ExcelReaderUtils {
             }
             result.add(map);
         }
+        list.clear();
         try {
             if (result.size() > 0) {
                 writeToExcel(result, filePath);