|
@@ -5,6 +5,7 @@ import com.skyversation.poiaddr.addquery.Constant;
|
|
|
import com.skyversation.poiaddr.bean.AddressResult;
|
|
|
import com.skyversation.poiaddr.config.DbConnection;
|
|
|
import com.skyversation.poiaddr.entity.SjArrDzbzhSjWcbryDzxx;
|
|
|
+import com.skyversation.poiaddr.util.SerializationUtils;
|
|
|
import com.skyversation.poiaddr.util.status.AddressLevel;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
@@ -20,6 +21,59 @@ import java.util.concurrent.Future;
|
|
|
@Service
|
|
|
public class SjArrDzbzhSjWcbryDzxxServiceImpl {
|
|
|
|
|
|
+ public List<SjArrDzbzhSjWcbryDzxx> getAllData(Integer PageSize, Integer PageNumber) {
|
|
|
+ try {
|
|
|
+ List<SjArrDzbzhSjWcbryDzxx> requestData = new ArrayList<>();
|
|
|
+ StringBuilder sql = new StringBuilder();
|
|
|
+ sql.append("SELECT * FROM sj_zrr_dzbzh_sj_wcbry_dzxx");
|
|
|
+ if (PageSize != null && PageNumber != null) {
|
|
|
+ sql.append(" order by id limit ").append(PageSize).append(" offset ").append((PageNumber - 1) * PageSize);
|
|
|
+ }
|
|
|
+ List<Map<String, Object>> oldDbData = DbConnection.getInstance().runSqlStr(sql.toString());
|
|
|
+// 去重
|
|
|
+ Map<String, Map<String, Object>> newRequestData = new HashMap<>();
|
|
|
+ for (Map<String, Object> item : oldDbData) {
|
|
|
+ if (item.get("id") != null) {
|
|
|
+ newRequestData.put(item.get("id").toString(), item);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ List<Map<String, Object>> dbData = new ArrayList<>();
|
|
|
+ for (String id : newRequestData.keySet()) {
|
|
|
+ dbData.add(newRequestData.get(id));
|
|
|
+ }
|
|
|
+
|
|
|
+ for (Map<String, Object> item : dbData) {
|
|
|
+ SjArrDzbzhSjWcbryDzxx sjArrDzbzhSjWcbryDzxx = new SjArrDzbzhSjWcbryDzxx();
|
|
|
+ if (item.get("id") != null) {
|
|
|
+ sjArrDzbzhSjWcbryDzxx.setId(item.getOrDefault("id", "").toString());
|
|
|
+ }
|
|
|
+ if (item.get("jc_hjdz") != null) {
|
|
|
+ sjArrDzbzhSjWcbryDzxx.setJcHjdz(item.getOrDefault("jc_hjdz", "").toString());
|
|
|
+ }
|
|
|
+ if (item.get("jc_jzdz") != null) {
|
|
|
+ sjArrDzbzhSjWcbryDzxx.setJcJzdz(item.getOrDefault("jc_jzdz", "").toString());
|
|
|
+ }
|
|
|
+ if (item.get("sk_hjdz") != null) {
|
|
|
+ sjArrDzbzhSjWcbryDzxx.setSkHjdz(item.getOrDefault("sk_hjdz", "").toString());
|
|
|
+ }
|
|
|
+ if (item.get("sk_jzdz") != null) {
|
|
|
+ sjArrDzbzhSjWcbryDzxx.setSkJzdz(item.getOrDefault("sk_jzdz", "").toString());
|
|
|
+ }
|
|
|
+ if (item.get("sqy_hjdz") != null) {
|
|
|
+ sjArrDzbzhSjWcbryDzxx.setSqyHjdz(item.getOrDefault("sqy_hjdz", "").toString());
|
|
|
+ }
|
|
|
+ if (item.get("sqy_jzdz") != null) {
|
|
|
+ sjArrDzbzhSjWcbryDzxx.setSqyJzdz(item.getOrDefault("sqy_jzdz", "").toString());
|
|
|
+ }
|
|
|
+ requestData.add(sjArrDzbzhSjWcbryDzxx);
|
|
|
+ }
|
|
|
+ return requestData;
|
|
|
+ } catch (Exception e) {
|
|
|
+ System.err.println("sql运行异常:" + e);
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
// 分页查询地址表
|
|
|
public List<SjArrDzbzhSjWcbryDzxx> getAllDataPage(int size) {
|
|
|
try {
|
|
@@ -32,7 +86,6 @@ public class SjArrDzbzhSjWcbryDzxxServiceImpl {
|
|
|
if (item.get("id") != null) {
|
|
|
newRequestData.put(item.get("id").toString(), item);
|
|
|
}
|
|
|
-
|
|
|
}
|
|
|
List<Map<String, Object>> dbData = new ArrayList<>();
|
|
|
for (String id : newRequestData.keySet()) {
|
|
@@ -182,6 +235,17 @@ public class SjArrDzbzhSjWcbryDzxxServiceImpl {
|
|
|
return true;
|
|
|
}
|
|
|
|
|
|
+ public void uploadAllData(int PageSize, int PageNumber) {
|
|
|
+ List<SjArrDzbzhSjWcbryDzxx> listData = getAllData(null, null);
|
|
|
+ if (listData != null && listData.size() > 0) {
|
|
|
+ SerializationUtils.serialize(runExecutorService(listData), "output/testDataBase_all.ser");
|
|
|
+// PageSize++;
|
|
|
+// uploadAllData(PageSize, PageNumber);
|
|
|
+// } else {
|
|
|
+// System.out.println("处理完成!");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 首先要得到一个Page<T>对象,然后判断是否还有别的数据,有的话接着请求并返回新的Page<T>对象并迭代处理数据
|
|
|
* init [0:批量处理;1:初始化]
|
|
@@ -192,98 +256,8 @@ public class SjArrDzbzhSjWcbryDzxxServiceImpl {
|
|
|
if (init == 0) {
|
|
|
List<SjArrDzbzhSjWcbryDzxx> listData = getAllDataPage(pageSize);
|
|
|
if (listData != null && listData.size() > 0) {
|
|
|
- // 创建线程池
|
|
|
- ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
|
|
- List<Future<?>> futures = new ArrayList<>();
|
|
|
- for (SjArrDzbzhSjWcbryDzxx item : listData) {
|
|
|
- futures.add(executorService.submit(() -> {
|
|
|
- List<String> addrList = new ArrayList<>();
|
|
|
- if (item.getSkHjdz() != null && !item.getSkHjdz().isEmpty() && isOtherDistrictThanSongJiang(item.getSkHjdz())) {
|
|
|
-// 实口户籍地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkHjdz()));
|
|
|
- }
|
|
|
- if (item.getSkJzdz() != null && !item.getSkJzdz().isEmpty() && isOtherDistrictThanSongJiang(item.getSkJzdz())) {
|
|
|
-// 实口居住地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkJzdz()));
|
|
|
- }
|
|
|
- if (item.getJcHjdz() != null && !item.getJcHjdz().isEmpty() && isOtherDistrictThanSongJiang(item.getJcHjdz())) {
|
|
|
-// 基层户籍地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcHjdz()));
|
|
|
- }
|
|
|
- if (item.getJcJzdz() != null && !item.getJcJzdz().isEmpty() && isOtherDistrictThanSongJiang(item.getJcJzdz())) {
|
|
|
-// 基层居住地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcJzdz()));
|
|
|
- }
|
|
|
-// 判断地址是否属于松江区
|
|
|
- if (addrList.size() > 0) {
|
|
|
-// 开始查询
|
|
|
- AddressResult addressResult = AddressQueryEngine.getInstance().commonSearchByName(addrList);
|
|
|
- if (addressResult == null || addressResult.getData() == null || addressResult.getData().size() < 1) {
|
|
|
- item.setInTheArea("异常");
|
|
|
- item.setResultAddrKey("未匹配到符合规则的结果");
|
|
|
- } else {
|
|
|
- try {
|
|
|
- for (AddressResult.ContentBean contentBean : addressResult.getData()) {
|
|
|
- String resultAddrKey = contentBean.getAddress();
|
|
|
- if (resultAddrKey != null && contentBean.getLon() != null && contentBean.getLat() != null && isOtherDistrictThanSongJiang(resultAddrKey)) {
|
|
|
- String lng = contentBean.getLon() + "";
|
|
|
- String lat = contentBean.getLat() + "";
|
|
|
- item.setInTheArea("是");
|
|
|
- if (contentBean.getAdname() != null && !contentBean.getAdname().isEmpty()) {
|
|
|
- item.setTwon(contentBean.getAdname());
|
|
|
- } else if (contentBean.getTownJson().getString("name") != null) {
|
|
|
- item.setTwon(contentBean.getTownJson().getString("name"));
|
|
|
- }
|
|
|
- if (contentBean.getCityname() != null && !contentBean.getCityname().isEmpty()) {
|
|
|
- item.setAdministrativeDivision(contentBean.getCityname());
|
|
|
- } else if (contentBean.getAdJson().getString("name") != null) {
|
|
|
- item.setAdministrativeDivision(contentBean.getAdJson().getString("name"));
|
|
|
- }
|
|
|
- item.setLat(lat);
|
|
|
- item.setLon(lng);
|
|
|
- item.setResultAddrKey(resultAddrKey);
|
|
|
- item.setMarket("上海市");
|
|
|
- item.setLevel(contentBean.getScore());
|
|
|
-// 开始规范化地址(把查询地址的市、区、街镇替换为空,然后把得到的区、街镇拼接上去)(得到区、街镇)
|
|
|
- String oldAddress = contentBean.getSearchAddress().replaceAll("上海市", "").replaceAll("松江区", "");
|
|
|
- String[] towns = Constant.getTowns();
|
|
|
- for (String town : towns) {
|
|
|
- oldAddress = oldAddress.replaceAll(town, "");
|
|
|
- }
|
|
|
- if (contentBean.getCjJson() != null && contentBean.getCjJson().containsKey("所属居委")) {
|
|
|
- oldAddress = contentBean.getCjJson().getString("所属居委") + oldAddress;
|
|
|
- }
|
|
|
- item.setStandardizedAddress("上海市" + item.getAdministrativeDivision() + item.getTwon() + oldAddress);
|
|
|
- break;
|
|
|
- } else {
|
|
|
- item.setInTheArea("异常");
|
|
|
- item.setResultAddrKey("结果处理异常");
|
|
|
- }
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- item.setInTheArea("异常");
|
|
|
- System.err.println("查询结果处理异常:" + e);
|
|
|
- }
|
|
|
- }
|
|
|
- } else {
|
|
|
-// 更新时间,添加是否松江判断字段为否
|
|
|
- item.setInTheArea("否");
|
|
|
- }
|
|
|
- item.setUpdateTime(new Date());
|
|
|
- }));
|
|
|
- }
|
|
|
-
|
|
|
- // 等待所有任务完成
|
|
|
- for (Future<?> future : futures) {
|
|
|
- try {
|
|
|
- future.get();
|
|
|
- } catch (InterruptedException | ExecutionException e) {
|
|
|
- System.err.println("线程异常:" + e);
|
|
|
- }
|
|
|
- }
|
|
|
- // 关闭线程池
|
|
|
- executorService.shutdown();
|
|
|
- updateDatas(listData);
|
|
|
+// 批量更新处理后的数据
|
|
|
+ updateDatas(runExecutorService(listData));
|
|
|
long endTime = System.currentTimeMillis();
|
|
|
System.out.println("处理单批次用时" + (endTime - startTime) / 1000 + "秒!");
|
|
|
if (listData.size() > 0) {
|
|
@@ -309,4 +283,97 @@ public class SjArrDzbzhSjWcbryDzxxServiceImpl {
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+ public List<SjArrDzbzhSjWcbryDzxx> runExecutorService(List<SjArrDzbzhSjWcbryDzxx> listData) {
|
|
|
+ // 创建线程池
|
|
|
+ ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
|
|
+ List<Future<?>> futures = new ArrayList<>();
|
|
|
+ for (SjArrDzbzhSjWcbryDzxx item : listData) {
|
|
|
+ futures.add(executorService.submit(() -> {
|
|
|
+ List<String> addrList = new ArrayList<>();
|
|
|
+ if (item.getSkHjdz() != null && !item.getSkHjdz().isEmpty() && isOtherDistrictThanSongJiang(item.getSkHjdz())) {
|
|
|
+ // 实口户籍地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkHjdz()));
|
|
|
+ }
|
|
|
+ if (item.getSkJzdz() != null && !item.getSkJzdz().isEmpty() && isOtherDistrictThanSongJiang(item.getSkJzdz())) {
|
|
|
+ // 实口居住地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkJzdz()));
|
|
|
+ }
|
|
|
+ if (item.getJcHjdz() != null && !item.getJcHjdz().isEmpty() && isOtherDistrictThanSongJiang(item.getJcHjdz())) {
|
|
|
+ // 基层户籍地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcHjdz()));
|
|
|
+ }
|
|
|
+ if (item.getJcJzdz() != null && !item.getJcJzdz().isEmpty() && isOtherDistrictThanSongJiang(item.getJcJzdz())) {
|
|
|
+ // 基层居住地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcJzdz()));
|
|
|
+ }
|
|
|
+ if (addrList.size() > 0) {
|
|
|
+ // TODO 开始查询
|
|
|
+ AddressResult addressResult = AddressQueryEngine.getInstance().commonSearchByName(addrList);
|
|
|
+ if (addressResult == null || addressResult.getData() == null || addressResult.getData().size() < 1) {
|
|
|
+ item.setInTheArea("异常");
|
|
|
+ item.setResultAddrKey("未匹配到符合规则的结果");
|
|
|
+ } else {
|
|
|
+ try {
|
|
|
+ for (AddressResult.ContentBean contentBean : addressResult.getData()) {
|
|
|
+ String resultAddrKey = contentBean.getAddress();
|
|
|
+ if (resultAddrKey != null && contentBean.getLon() != null && contentBean.getLat() != null && isOtherDistrictThanSongJiang(resultAddrKey)) {
|
|
|
+ String lng = contentBean.getLon() + "";
|
|
|
+ String lat = contentBean.getLat() + "";
|
|
|
+ item.setInTheArea("是");
|
|
|
+ if (contentBean.getAdname() != null && !contentBean.getAdname().isEmpty()) {
|
|
|
+ item.setTwon(contentBean.getAdname());
|
|
|
+ } else if (contentBean.getTownJson().getString("name") != null) {
|
|
|
+ item.setTwon(contentBean.getTownJson().getString("name"));
|
|
|
+ }
|
|
|
+ if (contentBean.getCityname() != null && !contentBean.getCityname().isEmpty()) {
|
|
|
+ item.setAdministrativeDivision(contentBean.getCityname());
|
|
|
+ } else if (contentBean.getAdJson().getString("name") != null) {
|
|
|
+ item.setAdministrativeDivision(contentBean.getAdJson().getString("name"));
|
|
|
+ }
|
|
|
+ item.setLat(lat);
|
|
|
+ item.setLon(lng);
|
|
|
+ item.setResultAddrKey(resultAddrKey);
|
|
|
+ item.setMarket("上海市");
|
|
|
+ item.setLevel(contentBean.getScore());
|
|
|
+// 开始规范化地址(把查询地址的市、区、街镇替换为空,然后把得到的区、街镇拼接上去)(得到区、街镇)
|
|
|
+ String oldAddress = contentBean.getSearchAddress().replaceAll("上海市", "").replaceAll("松江区", "");
|
|
|
+ String[] towns = Constant.getTowns();
|
|
|
+ for (String town : towns) {
|
|
|
+ oldAddress = oldAddress.replaceAll(town, "");
|
|
|
+ }
|
|
|
+ if (contentBean.getCjJson() != null && contentBean.getCjJson().containsKey("所属居委")) {
|
|
|
+ oldAddress = contentBean.getCjJson().getString("所属居委") + oldAddress;
|
|
|
+ }
|
|
|
+ item.setStandardizedAddress("上海市" + item.getAdministrativeDivision() + item.getTwon() + oldAddress);
|
|
|
+ break;
|
|
|
+ } else {
|
|
|
+ item.setInTheArea("异常");
|
|
|
+ item.setResultAddrKey("结果处理异常");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ item.setInTheArea("异常");
|
|
|
+ System.err.println("查询结果处理异常:" + e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ item.setInTheArea("否");
|
|
|
+ }
|
|
|
+ item.setUpdateTime(new Date());
|
|
|
+ }));
|
|
|
+ }
|
|
|
+
|
|
|
+ // 等待所有任务完成
|
|
|
+ for (Future<?> future : futures) {
|
|
|
+ try {
|
|
|
+ future.get();
|
|
|
+ } catch (InterruptedException | ExecutionException e) {
|
|
|
+ System.err.println("线程异常:" + e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 关闭线程池
|
|
|
+ executorService.shutdown();
|
|
|
+ return listData;
|
|
|
+ }
|
|
|
}
|