更新分词器；添加从高德接口获取数据库失败数据的数据并回流的方法；更新jpa对象适应callback表

src/main/java/com/skyversation/poiaddr/addquery/AddressQueryEngine.java

@@ -6,7 +6,7 @@ import com.skyversation.poiaddr.bean.AddressResult;
 import com.skyversation.poiaddr.bean.GeoJsonBean;
 import com.skyversation.poiaddr.bean.WDToken;
 import com.skyversation.poiaddr.service.AreaService;
-import com.skyversation.poiaddr.util.ExcelReaderUtils;
+import com.skyversation.poiaddr.service.DataCallBackService;
 import com.skyversation.poiaddr.util.net.AddressNetTools;
 import com.skyversation.poiaddr.util.status.AddressLevel;
 import com.skyversation.poiaddr.util.status.AddressResultEnum;
@@ -17,7 +17,6 @@ import org.springframework.util.StringUtils;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
-import java.io.IOException;
 import java.util.*;
 
 import static com.skyversation.poiaddr.addquery.TransfromDataTool.yyszResultToResult;
@@ -30,7 +29,7 @@ public class AddressQueryEngine {
     @Value("${app.yysz-address-service}")
     private String yyszAddressService=null;
     @Resource
-    private DataCallBack dataCallBack;
+    private DataCallBackService dataCallBack;
     @PostConstruct
     public void init(){
         if (netType == null) {
@@ -246,7 +245,7 @@ public class AddressQueryEngine {
     }/**
      * 高德高级多条搜索
      */
-    private AddressResult gdV3MultiSearch(List<String> addrs, AddressLevel level) {
+    public AddressResult gdV3MultiSearch(List<String> addrs, AddressLevel level) {
         try {
             for (String addr : addrs) {
                 AddressResult.ContentBean contentBean = verificaData(gdV3SearchByName(addr), level, addr);

src/main/java/com/skyversation/poiaddr/util/jpa/entity/TAddressCallback.java → src/main/java/com/skyversation/poiaddr/bean/TAddressCallback.java

@@ -1,4 +1,4 @@
-package com.skyversation.poiaddr.util.jpa.entity;
+package com.skyversation.poiaddr.bean;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -8,7 +8,6 @@ import org.springframework.data.domain.Persistable;
 
 import javax.persistence.*;
 import java.time.Instant;
-import java.time.LocalDate;
 
 @Getter
 @Setter
@@ -45,6 +44,9 @@ public class TAddressCallback implements Persistable<Integer> {
     @Column(name = "status")
     private Short status;
 
+    @Column(name = "process_status")
+    private Short processStatus;
+
     @Override
     @Transient
     public boolean isNew() {

src/main/java/com/skyversation/poiaddr/util/jpa/entity/YyszAddressQp.java → src/main/java/com/skyversation/poiaddr/bean/YyszAddressQp.java

@@ -1,6 +1,5 @@
-package com.skyversation.poiaddr.util.jpa.entity;
+package com.skyversation.poiaddr.bean;
 
-import com.skyversation.poiaddr.util.jpa.rep.YyszAddressQpRep;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
@@ -10,7 +9,6 @@ import org.hibernate.annotations.Type;
 import org.springframework.data.domain.Persistable;
 
 import javax.persistence.*;
-import java.sql.SQLException;
 import java.time.Instant;
 
 @Getter

src/main/java/com/skyversation/poiaddr/repository/TAddressCallbackRep.java

@@ -0,0 +1,21 @@
+package com.skyversation.poiaddr.repository;
+
+import com.skyversation.poiaddr.bean.TAddressCallback;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+
+import java.util.Optional;
+
+public interface TAddressCallbackRep extends JpaRepository<TAddressCallback,Integer> {
+    @Query(value = "SELECT MAX(e.id) FROM t_address_callback e", nativeQuery = true)
+    Optional<Integer> findMaxId();
+    @Query(value = "SELECT DISTINCT t.address FROM t_address_callback t where t.process_status = 0 or t.process_status = 1",
+            countQuery = "SELECT COUNT(DISTINCT a.address) FROM t_address_callback a WHERE t.process_status = 0 or t.process_status = 1",nativeQuery = true)
+    Page<String> findAllInNative(Pageable pageable);
+    @Query(value = "update t_address_callback set process_status = :status where address = :target",nativeQuery = true)
+    @Modifying
+    void updateAllProcessStatus(String target, Integer status);
+}

src/main/java/com/skyversation/poiaddr/util/jpa/rep/YyszAddressQpRep.java → src/main/java/com/skyversation/poiaddr/repository/YyszAddressQpRep.java

@@ -1,6 +1,6 @@
-package com.skyversation.poiaddr.util.jpa.rep;
+package com.skyversation.poiaddr.repository;
 
-import com.skyversation.poiaddr.util.jpa.entity.YyszAddressQp;
+import com.skyversation.poiaddr.bean.YyszAddressQp;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;

src/main/java/com/skyversation/poiaddr/addquery/DataCallBack.java → src/main/java/com/skyversation/poiaddr/service/DataCallBackService.java

@@ -1,37 +1,47 @@
-package com.skyversation.poiaddr.addquery;
-
+package com.skyversation.poiaddr.service;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.skyversation.poiaddr.addquery.AddressQueryEngine;
+import com.skyversation.poiaddr.addquery.AddressTools;
+import com.skyversation.poiaddr.addquery.Constant;
+import com.skyversation.poiaddr.addquery.TransfromDataTool;
 import com.skyversation.poiaddr.bean.AddressResult;
 import com.skyversation.poiaddr.bean.GeoJsonBean;
 import com.skyversation.poiaddr.util.address_spliter.SplitAddress;
-import com.skyversation.poiaddr.service.AreaService;
 import com.skyversation.poiaddr.util.CoordTransform2;
 import com.skyversation.poiaddr.util.address_spliter.ShanghaiAddressSplitUtil;
-import com.skyversation.poiaddr.util.jpa.entity.TAddressCallback;
-import com.skyversation.poiaddr.util.jpa.entity.YyszAddressQp;
-import com.skyversation.poiaddr.util.jpa.rep.TAddressCallbackRep;
-import com.skyversation.poiaddr.util.jpa.rep.YyszAddressQpRep;
+import com.skyversation.poiaddr.bean.TAddressCallback;
+import com.skyversation.poiaddr.bean.YyszAddressQp;
+import com.skyversation.poiaddr.repository.TAddressCallbackRep;
+import com.skyversation.poiaddr.repository.YyszAddressQpRep;
+import com.skyversation.poiaddr.util.net.AddressNetTools;
+import com.skyversation.poiaddr.util.status.AddressLevel;
+import com.skyversation.poiaddr.util.status.AddressResultEnum;
 import lombok.Getter;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.TransactionStatus;
 import org.springframework.transaction.support.TransactionCallbackWithoutResult;
 import org.springframework.transaction.support.TransactionTemplate;
+import org.springframework.util.StringUtils;
 
 import javax.annotation.Resource;
 import java.time.Instant;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 @Service
-public class DataCallBack {
-    private DataCallBack(){};
+public class DataCallBackService {
+    private DataCallBackService(){};
     @Getter
-    private static final DataCallBack instance = new DataCallBack();
+    private static final DataCallBackService instance = new DataCallBackService();
     private final ExecutorService successES = Executors.newSingleThreadExecutor();
     private final ExecutorService failES = Executors.newSingleThreadExecutor();
     @Resource
@@ -40,6 +50,8 @@ public class DataCallBack {
     private YyszAddressQpRep yyszAddressQpRep;
     @Resource
     private ShanghaiAddressSplitUtil shanghaiAddressSplitUtil;
+    @Resource
+    private AddressQueryEngine addressQueryEngine;
 
     @Resource
     private PlatformTransactionManager transactionManager;
@@ -57,6 +69,9 @@ public class DataCallBack {
         CHINESE_TO_ARABIC.put("第九", "09");
     }
 
+    /**
+     * 成功数据入库
+     */
     public void successDataCallBack(AddressResult addressResult){
         for (AddressResult.ContentBean bean :addressResult.getData()){
             successES.execute(() -> {
@@ -153,6 +168,10 @@ public class DataCallBack {
         }
 
     }
+
+    /**
+     * 失败数据入库
+     */
     public void failDataCallBack(List<String> addr){
         for (String addrStr:addr){
             failES.execute(()->{
@@ -165,11 +184,46 @@ public class DataCallBack {
                 int id = tAddressCallbackRep.findMaxId().orElse(0)+1;
                 TAddressCallback tAddressCallback = new TAddressCallback(id, Instant.now(),addrStr, splitAddress.getAddr(),
                         splitAddress.getProvince(),splitAddress.getCity(),splitAddress.getStreet(), splitAddress.getCommunity(),
-                        (short) 0);
+                        (short) 0, (short) 0);
                 tAddressCallbackRep.save(tAddressCallback);
                 System.out.println("回流成功,等待进一检查处理");
             });
         }
+    }
+    /**
+     * 用于将失败数据调高德接口回流
+     */
+    public void dbFailDataCallBackTask(){
+        int pageNum = 0;
+        int pageSize = 1000;
+        Page<String> page;
+        TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
+        do {
+            page = tAddressCallbackRep.findAllInNative(PageRequest.of(pageNum,pageSize ));
+            pageNum++;
+            for (String tAddressCallback : page.getContent()) {
 
+                transactionTemplate.execute(new TransactionCallbackWithoutResult() {
+                    @Override
+                    protected void doInTransactionWithoutResult(TransactionStatus status) {
+                        try {
+                            System.out.println("尝试从szx接口获取地址:" + tAddressCallback);
+                            AddressResult addressResult = addressQueryEngine.gdV3MultiSearch(Collections.singletonList(tAddressCallback),AddressLevel.LEVEL_1);
+                            if (addressResult ==null) {
+                                System.out.println("无结果:" + tAddressCallback);
+                                tAddressCallbackRep.updateAllProcessStatus(tAddressCallback,1);
+                            }else {
+                                System.out.println("有结果，回流:" + tAddressCallback);
+                                tAddressCallbackRep.updateAllProcessStatus(tAddressCallback,2);
+                                successDataCallBack(addressResult);
+                            }
+                        } catch (Exception e) {
+                            status.setRollbackOnly();
+                            e.printStackTrace();
+                        }
+                    }
+                });
+            }
+        } while (page.hasNext());
     }
 }

src/main/java/com/skyversation/poiaddr/util/address_spliter/AddressSplitUtil.java

@@ -1,5 +1,6 @@
 package com.skyversation.poiaddr.util.address_spliter;
 
+
 import com.skyversation.poiaddr.util.ExcelReaderUtils;
 
 import java.io.InputStream;
@@ -33,7 +34,7 @@ public class AddressSplitUtil {
 
     private static final Pattern LEVEL_3_SUFFIX_PATTERN = Pattern.compile("^(?:县|自治县|市|区|旗|自治旗|林区|特区)");
 
-    public static final Pattern ROAD_SUFFIX_PATTERN = Pattern.compile("^(?:旅游区|[东南西北中一二三四五六七八九十公大小支新老]{0,2}(?:大街|路|环路|大道|街|菜市场|马路|村))");
+    public static final Pattern ROAD_SUFFIX_PATTERN = Pattern.compile("^(?:旅游区|[东南西北中一二三四五六七八九十公大小支新老]{0,2}(?:大街|环路|路|大道|街|菜市场|马路|村))");
 
     static {
         Map<String,String> level1NameMap = new HashMap<>();

src/main/java/com/skyversation/poiaddr/util/address_spliter/ShanghaiAddressSplitUtil.java

@@ -317,10 +317,10 @@ public class ShanghaiAddressSplitUtil {
         int  disIndex= washResult(beautyAddress, districtContainResult,LEVEL_1_SUFFIX_PATTERN);
         Map<Integer, String> streetContainResult = contain(beautyAddress, STREET_TO_COMMUNITY_MAP.keySet(), 0);
         int  streetIndex= washResult(beautyAddress, districtContainResult,LEVEL_2_SUFFIX_PATTERN);
+        splitAddress.setAddr(result[3]);
         if (!((result[0].isEmpty()|| result[0].equals("上海市")) && (result[1].isEmpty()  || result[1].equals("上海市"))||
                 !districtContainResult.isEmpty()||!streetContainResult.isEmpty())) {
             splitAddress.setStatus(2);
-            splitAddress.setAddr(result[3]);
             return splitAddress;
         }
         if (!districtContainResult.isEmpty()){
@@ -345,6 +345,8 @@ public class ShanghaiAddressSplitUtil {
         if (splittingAddress.street!=-1||splittingAddress.community!=-1){
             splitAddress.setStreet(splittingAddress.threeLevelAddress.streetFullName);
             splitAddress.setStreetCode(splittingAddress.threeLevelAddress.streetCode);
+            splitAddress.setDistrict(splittingAddress.threeLevelAddress.districtFullName);
+            splitAddress.setDistrictCode(splittingAddress.threeLevelAddress.districtCode);
 
         }
         if (splittingAddress.community!=-1){
@@ -352,10 +354,6 @@ public class ShanghaiAddressSplitUtil {
             splitAddress.setCommunityCode(splittingAddress.threeLevelAddress.communityCode);
 
         }
-        if (disIndex==-1){
-            splitAddress.setDistrict(splittingAddress.threeLevelAddress.districtFullName);
-            splitAddress.setDistrictCode(splittingAddress.threeLevelAddress.districtCode);
-        }
 
 
         //检查是否能够分离

src/main/java/com/skyversation/poiaddr/util/address_spliter/SplitAddress.java

@@ -5,7 +5,7 @@ import lombok.Data;
 /**
  * 分割后的数据
  * @see SplitAddress#status
- * @see com/skyversation/poiaddr.util.ShanghaiAddressSplitUtil
+ * @see ShanghaiAddressSplitUtil
  */
 @Data
 public class SplitAddress implements Comparable<SplitAddress>{
@@ -18,7 +18,7 @@ public class SplitAddress implements Comparable<SplitAddress>{
      * 3->非地址（检查到源地址非正常地址,例如网址）<br/>
      * 4->无法确认是否为外省
      */
-    private int status=-1;
+    private int status;
     private String addr = "";
     private String province = "";
     private String city = "";
@@ -29,6 +29,7 @@ public class SplitAddress implements Comparable<SplitAddress>{
     private String districtCode = "";
     private String streetCode = "";
     private String communityCode="";
+    private String searchAddr;
     public boolean isSuccess(){
         switch (status){
             case 0: case 1: case 4:{

src/main/java/com/skyversation/poiaddr/util/jpa/rep/TAddressCallbackRep.java

@@ -1,12 +0,0 @@
-package com.skyversation.poiaddr.util.jpa.rep;
-
-import com.skyversation.poiaddr.util.jpa.entity.TAddressCallback;
-import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Query;
-
-import java.util.Optional;
-
-public interface TAddressCallbackRep extends JpaRepository<TAddressCallback,Integer> {
-    @Query(value = "SELECT MAX(e.id) FROM t_address_callback e", nativeQuery = true)
-    Optional<Integer> findMaxId();
-}