|
@@ -2,16 +2,13 @@ package com.skyversation.poiaddr.service;
|
|
|
|
|
|
import com.alibaba.fastjson.JSONArray;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
-import com.skyversation.poiaddr.addquery.AddressQueryEngine;
|
|
|
import com.skyversation.poiaddr.addquery.Constant;
|
|
|
import com.skyversation.poiaddr.bean.*;
|
|
|
import com.skyversation.poiaddr.config.DbConnection;
|
|
|
import com.skyversation.poiaddr.service.impl.TAddressCallbackRepository;
|
|
|
import com.skyversation.poiaddr.service.impl.YyszAddressRepository;
|
|
|
-import com.skyversation.poiaddr.service.impl.ZrrCallBackRepository;
|
|
|
import com.skyversation.poiaddr.util.ShanghaiAddressSplitUtil;
|
|
|
import com.skyversation.poiaddr.util.SplitAddress;
|
|
|
-import com.skyversation.poiaddr.util.status.AddressResultEnum;
|
|
|
import com.skyversation.poiaddr.util.tasks.ScheduledTasks;
|
|
|
import org.geotools.geojson.geom.GeometryJSON;
|
|
|
import org.geotools.geometry.jts.JTSFactoryFinder;
|
|
@@ -22,16 +19,13 @@ import org.locationtech.jts.io.WKTReader;
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
import org.springframework.core.io.ClassPathResource;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
+import org.springframework.transaction.annotation.Transactional;
|
|
|
import org.springframework.util.StringUtils;
|
|
|
|
|
|
import javax.annotation.PostConstruct;
|
|
|
import javax.annotation.Resource;
|
|
|
import java.io.*;
|
|
|
import java.util.*;
|
|
|
-import java.util.concurrent.ExecutionException;
|
|
|
-import java.util.concurrent.ExecutorService;
|
|
|
-import java.util.concurrent.Executors;
|
|
|
-import java.util.concurrent.Future;
|
|
|
|
|
|
@Service
|
|
|
public class AreaService {
|
|
@@ -42,9 +36,6 @@ public class AreaService {
|
|
|
@Resource
|
|
|
private YyszAddressRepository yyszAddressRepository;
|
|
|
|
|
|
- @Resource
|
|
|
- private ZrrCallBackRepository zrrCallBackRepository;
|
|
|
-
|
|
|
@Resource
|
|
|
private TAddressCallbackRepository tAddressCallbackRepository;
|
|
|
|
|
@@ -91,7 +82,7 @@ public class AreaService {
|
|
|
System.out.println("<<<<<<<<------根据配置文件为AREA赋值:" + Constant.getArea());
|
|
|
System.out.println("<<<<<<<<------当前AMAP_CITY_CODE:" + Constant.getAMAP_CITY_CODE());
|
|
|
// TODO 网络连通性测试,可以请求一下测试的市中心地址
|
|
|
- try {
|
|
|
+ /*try {
|
|
|
System.out.println("电脑最大线程数:" + Runtime.getRuntime().availableProcessors());
|
|
|
System.out.println("<<<<<<<<------开始网络连通性测试");
|
|
|
long startTime = System.currentTimeMillis();
|
|
@@ -102,12 +93,13 @@ public class AreaService {
|
|
|
System.out.println(">>>>>>>>------网络连通性测试完成!用时" + (endTime - startTime) / 1000 + "秒!结果:" + addressResult);
|
|
|
} catch (Exception e) {
|
|
|
System.err.println(">>>>>>>>------网络连通性测试结果:" + e);
|
|
|
- }
|
|
|
+ }*/
|
|
|
// TODO 数据库连通性测试
|
|
|
try {
|
|
|
- System.out.println("<<<<<<<<------开始数据库连通性测试");
|
|
|
- System.out.println("------数据库连通性测试结果:" + DbConnection.getInstance().runSqlStr("select count(*) from " + searchTableName));
|
|
|
- initScheduledTasksSetAddress();
|
|
|
+ /*System.out.println("<<<<<<<<------开始数据库连通性测试");
|
|
|
+ System.out.println("------数据库连通性测试结果:" + DbConnection.getInstance().runSqlStr("select count(*) from " + searchTableName));*/
|
|
|
+// System.out.println("<<<<<<<<------开始缓存待处理表对象");
|
|
|
+// initScheduledTasksSetAddress();
|
|
|
} catch (Exception e) {
|
|
|
System.err.println(">>>>>>>>------数据库连通性测试结果:" + e);
|
|
|
}
|
|
@@ -119,20 +111,9 @@ public class AreaService {
|
|
|
ScheduledTasks.callBackAllErrorAddrs.add(tAddressCallback.getSearchAddress());
|
|
|
}
|
|
|
}
|
|
|
- /*List<Map<String, Object>> callBackAddr = DbConnection.getInstance().runSqlStr("select * from t_address_callback");
|
|
|
- if (callBackAddr != null && callBackAddr.size() == 1) {
|
|
|
- for (Map<String, Object> item : callBackAddr) {
|
|
|
- Integer maxId = Integer.parseInt(item.get("id").toString());
|
|
|
- if (ScheduledTasks.callBackMaxId < maxId) {
|
|
|
- ScheduledTasks.callBackMaxId = maxId;
|
|
|
- }
|
|
|
- ScheduledTasks.callBackAllErrorAddrs.add(item.get("address").toString());
|
|
|
- }
|
|
|
- }*/
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
-
|
|
|
}
|
|
|
|
|
|
public void initPolygons() {
|
|
@@ -184,112 +165,76 @@ public class AreaService {
|
|
|
System.out.println("<<<<<<<<------run geo data complete------>>>>>>>>>");
|
|
|
}
|
|
|
|
|
|
- public void initScheduledTasksSetAddress() {
|
|
|
+ public void putAllDmdzData(String addr, YyskDmdzAddressStandardization item) {
|
|
|
+ if (StringUtils.hasText(addr)) {
|
|
|
+ if (addr.contains("号")) {
|
|
|
+ addr = addr.substring(0, addr.indexOf("号"));
|
|
|
+ }
|
|
|
+ if (addr.contains("弄")) {
|
|
|
+ addr = addr.substring(0, addr.indexOf("弄"));
|
|
|
+ }
|
|
|
+ if (!ScheduledTasks.allDmdzData.containsKey(addr)) {
|
|
|
+ List<YyskDmdzAddressStandardization> datas = new ArrayList<>();
|
|
|
+ datas.add(item);
|
|
|
+ ScheduledTasks.allDmdzData.put(addr, datas);
|
|
|
+ } else if (ScheduledTasks.allDmdzData.get(addr).size() < 20) {
|
|
|
+ ScheduledTasks.allDmdzData.get(addr).add(item);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 缓存地址库数据
|
|
|
+ */
|
|
|
+ public void getAllDmdzAddressDatas() {
|
|
|
long startTime = System.currentTimeMillis();
|
|
|
- List<ZrrCallback20250609> zrrCallback20250609s = zrrCallBackRepository.findAll();
|
|
|
- if (zrrCallback20250609s.size() > 0) {
|
|
|
- for (ZrrCallback20250609 zrrCallback20250609 : zrrCallback20250609s) {
|
|
|
- ScheduledTasks.setAddress.add(zrrCallback20250609.getSearchAddress());
|
|
|
+// 先查询到所有数据
|
|
|
+ getDmdzDataByPage(0, 10000);
|
|
|
+ System.out.println("ScheduledTasks.allDmdzData初始化完成,有" + ScheduledTasks.allDmdzData.size() + "条记录保存到缓存中,用时" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * JPA的形式加载所有地址信息到缓存
|
|
|
+ * @param maxOid
|
|
|
+ * @param pageSize
|
|
|
+ */
|
|
|
+ @Transactional(readOnly = true)
|
|
|
+ public void getDmdzDataByPage(int maxOid, int pageSize) {
|
|
|
+ System.out.println("maxOid:" + maxOid + "pageSize:" + pageSize);
|
|
|
+ long sTime = System.currentTimeMillis();
|
|
|
+ List<YyskDmdzAddressStandardization> dataList = yyszAddressRepository.getAllByOidPage(maxOid, pageSize);
|
|
|
+ System.out.println("当前OID" + maxOid + "查询完成,用时:" + (System.currentTimeMillis() - sTime) / 1000 + "秒!");
|
|
|
+ // TODO 经测试发现一次性读全表,内存根本吃不消
|
|
|
+ for (YyskDmdzAddressStandardization item : dataList) {
|
|
|
+ if (item.getAddress() != null && StringUtils.hasText(item.getAddress())) {
|
|
|
+ SplitAddress splitAddress = ShanghaiAddressSplitUtil.splitBestAddress(item.getAddress());
|
|
|
+ putAllDmdzData(splitAddress.getAddr(), item);
|
|
|
+ }
|
|
|
+ if (item.getSourceaddress() != null && StringUtils.hasText(item.getSourceaddress())) {
|
|
|
+ SplitAddress splitAddress = ShanghaiAddressSplitUtil.splitBestAddress(item.getSourceaddress());
|
|
|
+ putAllDmdzData(splitAddress.getAddr(), item);
|
|
|
}
|
|
|
+ if (item.getOid() > maxOid) {
|
|
|
+ maxOid = item.getOid();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ System.out.println("当前OID" + maxOid + "查询处理完成,用时:" + (System.currentTimeMillis() - sTime) / 1000 + "秒!");
|
|
|
+ if (dataList.size() > 0) {
|
|
|
+ getDmdzDataByPage(maxOid, pageSize);
|
|
|
+ } else {
|
|
|
+ System.out.println("分页查询结束");
|
|
|
}
|
|
|
- System.out.println("ScheduledTasks.setAddress初始化完成,共" + ScheduledTasks.setAddress.size() + "条,查询用时:" + (System.currentTimeMillis() - startTime) / 1000 + "秒!");
|
|
|
}
|
|
|
|
|
|
- public void selectLimitData(int page, int pageSize, int endPage, String par_dt) {
|
|
|
+ public void selectLimitData(int page, int pageSize) {
|
|
|
Map<String, Object> loginfoMap = new HashMap<>();
|
|
|
loginfoMap.put("page", page);
|
|
|
loginfoMap.put("pageSize", pageSize);
|
|
|
try {
|
|
|
long startTime = System.currentTimeMillis();
|
|
|
loginfoMap.put("startTime", startTime);
|
|
|
- List<Map<String, Object>> callBackDatas = DbConnection.getInstance().runSqlStr("select roomdetailedaddress,hjdz from dws.dws_sjqdsjzx_zrr_hjjzxx_arrange where par_dt = '" + par_dt + "' order by ryid limit " + (page * pageSize) + "," + pageSize);
|
|
|
- System.out.println("数据库查询par_dt:" + par_dt + ",page:" + page + ",pageSize:" + pageSize + "完成,用时:" + (System.currentTimeMillis() - startTime) / 1000 + "秒!");
|
|
|
- if (callBackDatas != null && callBackDatas.size() > 0 && page < endPage) {
|
|
|
- Set<String> address = new HashSet<>();
|
|
|
- for (Map<String, Object> dataI : callBackDatas) {
|
|
|
- if (dataI.get("roomdetailedaddress") != null && StringUtils.hasText(dataI.get("roomdetailedaddress") + "") && !ScheduledTasks.setAddress.contains(dataI.get("roomdetailedaddress") + "")) {
|
|
|
- address.add(dataI.get("roomdetailedaddress") + "");
|
|
|
- ScheduledTasks.setAddress.add(dataI.get("roomdetailedaddress") + "");
|
|
|
- }
|
|
|
- if (dataI.get("hjdz") != null && StringUtils.hasText(dataI.get("hjdz") + "") && !ScheduledTasks.setAddress.contains(dataI.get("hjdz") + "")) {
|
|
|
- address.add(dataI.get("hjdz") + "");
|
|
|
- ScheduledTasks.setAddress.add(dataI.get("hjdz") + "");
|
|
|
- }
|
|
|
- }
|
|
|
- callBackDatas.clear();
|
|
|
- List<ZrrCallback20250609> outDatas = new ArrayList<>();
|
|
|
- int MaxTnumber = Runtime.getRuntime().availableProcessors();
|
|
|
- System.out.println("共:《" + address.size() + "》条数据!最大可用线程数:" + MaxTnumber);
|
|
|
- int index = 0;
|
|
|
- ExecutorService executorService = Executors.newFixedThreadPool(MaxTnumber / 2);
|
|
|
- List<Future<?>> futures = new ArrayList<>();
|
|
|
- for (String addrStr : address) {
|
|
|
- index++;
|
|
|
- int finalIndex = index;
|
|
|
- futures.add(executorService.submit(() -> {
|
|
|
- System.out.print(">" + finalIndex);
|
|
|
- ZrrCallback20250609 zrrCallback20250609 = new ZrrCallback20250609();
|
|
|
- zrrCallback20250609.setSearchAddress(addrStr);
|
|
|
- AddressResult addressResult = AddressQueryEngine.getInstance().commonSearchByName_nw(addrStr);
|
|
|
- AddressResult.ContentBean contentBean = new AddressResult.ContentBean();
|
|
|
- if (addressResult != null && addressResult.getCode() == AddressResultEnum.DB_SUCCESS && addressResult.getData() != null && addressResult.getData().size() > 0) {
|
|
|
- contentBean = addressResult.getData().get(0);
|
|
|
- }
|
|
|
- if (contentBean != null && contentBean.getLon() != null) {
|
|
|
- AddressQueryEngine.getInstance().getCjWgWgwByLoc(addressResult);
|
|
|
- zrrCallback20250609.setLat(contentBean.getLat() + "");
|
|
|
- zrrCallback20250609.setLon(contentBean.getLon() + "");
|
|
|
- zrrCallback20250609.setPname(contentBean.getPname());
|
|
|
- zrrCallback20250609.setCounty(contentBean.getCityname());
|
|
|
- zrrCallback20250609.setCankaoAddress(contentBean.getAddress());
|
|
|
- zrrCallback20250609.setStreetTownName(contentBean.getAdname());
|
|
|
- zrrCallback20250609.setCommunity(contentBean.getCommunity());
|
|
|
- zrrCallback20250609.setStandardAddress(contentBean.getStandAddr());
|
|
|
- SplitAddress splitAddress = ShanghaiAddressSplitUtil.splitBestAddress(addrStr);
|
|
|
- if (!StringUtils.hasText(contentBean.getPname())) {
|
|
|
- contentBean.setPname(splitAddress.getCity());
|
|
|
- zrrCallback20250609.setPname(splitAddress.getCity());
|
|
|
- }
|
|
|
- if (!StringUtils.hasText(contentBean.getPname())) {
|
|
|
- contentBean.setPname("上海市");
|
|
|
- zrrCallback20250609.setPname("上海市");
|
|
|
- }
|
|
|
- zrrCallback20250609.setStandardAddress(contentBean.getPname() + contentBean.getCityname() +
|
|
|
- contentBean.getAdname() + contentBean.getCommunity() + splitAddress.getAddr());
|
|
|
- }
|
|
|
- if (zrrCallback20250609.getSearchAddress() != null) {
|
|
|
- outDatas.add(zrrCallback20250609);
|
|
|
- }
|
|
|
- }));
|
|
|
- }
|
|
|
-
|
|
|
- // 等待所有任务完成
|
|
|
- for (Future<?> future : futures) {
|
|
|
- try {
|
|
|
- future.get();
|
|
|
- } catch (InterruptedException | ExecutionException e) {
|
|
|
- System.err.println("线程异常:" + e);
|
|
|
- }
|
|
|
- }
|
|
|
- // 关闭线程池
|
|
|
- executorService.shutdown();
|
|
|
- zrrCallBackRepository.saveAll(outDatas);
|
|
|
- long endTime = System.currentTimeMillis();
|
|
|
- System.out.println("处理完成!用时" + (endTime - startTime) / 1000 + "秒!");
|
|
|
- loginfoMap.put("endTime", endTime);
|
|
|
- loginfoMap.put("message", "成功");
|
|
|
- page++;
|
|
|
- selectLimitData(page, pageSize, endPage, par_dt);
|
|
|
- } else if (page < endPage) {
|
|
|
- loginfoMap.put("endTime", System.currentTimeMillis());
|
|
|
- loginfoMap.put("message", "当前页数据已处理!");
|
|
|
- page++;
|
|
|
- selectLimitData(page, pageSize, endPage, par_dt);
|
|
|
- } else {
|
|
|
- System.out.println("所有数据处理完成");
|
|
|
- loginfoMap.put("endTime", System.currentTimeMillis());
|
|
|
- loginfoMap.put("message", "所有数据处理完成!");
|
|
|
- }
|
|
|
+ System.out.println("数据库查询page:" + page + ",pageSize:" + pageSize + "完成,用时:" + (System.currentTimeMillis() - startTime) / 1000 + "秒!");
|
|
|
+// TODO 要使用JDBC的方式去查询待处理表的数据,然后有个sql需要执行一下,筛选出结果表中不存在的数据,然后进行下一步处理。
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|
|
|
loginfoMap.put("endTime", System.currentTimeMillis());
|
|
@@ -303,6 +248,7 @@ public class AreaService {
|
|
|
* 批量插入数据到callback表
|
|
|
*/
|
|
|
public void callBackAddrs() {
|
|
|
+// TOOD jpa
|
|
|
try {
|
|
|
List<TAddressCallback> tAddressCallbacks = tAddressCallbackRepository.saveAll(ScheduledTasks.callBackDatas);
|
|
|
if (tAddressCallbacks.size() > 0) {
|
|
@@ -313,7 +259,7 @@ public class AreaService {
|
|
|
e.printStackTrace();
|
|
|
System.out.println("数据回流异常:" + e);
|
|
|
}
|
|
|
-
|
|
|
+// TODO sql
|
|
|
/*StringBuilder sql = new StringBuilder("insert into t_address_callback (id,create_time,address,name,provice_name,city_name,town_name,community_name,status,process_status,is_new) values ");
|
|
|
for (String addr : ScheduledTasks.callBackAddrs) {
|
|
|
SplitAddress splitAddress = ShanghaiAddressSplitUtil.splitBestAddress(addr);
|
|
@@ -331,7 +277,7 @@ public class AreaService {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * 是否进入callBack表判断
|
|
|
+ * 是否写入callBack表判断
|
|
|
*
|
|
|
* @param addrStr
|
|
|
*/
|
|
@@ -432,7 +378,7 @@ public class AreaService {
|
|
|
}
|
|
|
return null;
|
|
|
}*/
|
|
|
- public List<yyskDmdzAddressStandardization> getAddressPoisByAddr(SplitAddress splitAddress) {
|
|
|
+ public List<YyskDmdzAddressStandardization> getAddressPoisByAddr(SplitAddress splitAddress) {
|
|
|
if (splitAddress.getAddr().contains("号")) {
|
|
|
splitAddress.setSearchAddr(splitAddress.getAddr().substring(0, splitAddress.getAddr().indexOf("号")));
|
|
|
}
|
|
@@ -443,22 +389,54 @@ public class AreaService {
|
|
|
if (splitAddress.getSearchAddr() == null) {
|
|
|
splitAddress.setSearchAddr(splitAddress.getAddr());
|
|
|
}
|
|
|
- List<yyskDmdzAddressStandardization> list = null;
|
|
|
+ List<YyskDmdzAddressStandardization> list = null;
|
|
|
if (StringUtils.hasText(splitAddress.getDistrict())) {
|
|
|
- list = yyszAddressRepository.getAddressPoisByAddrAQX(splitAddress.getDistrict(), splitAddress.getSearchAddr());
|
|
|
+// TODO 数据库查询逻辑调整为内存处理的方式
|
|
|
+ List<YyskDmdzAddressStandardization> dmdzList = new ArrayList<>();
|
|
|
+ if (ScheduledTasks.allDmdzData.containsKey(splitAddress.getSearchAddr())) {
|
|
|
+ dmdzList = ScheduledTasks.allDmdzData.get(splitAddress.getSearchAddr());
|
|
|
+ } else {
|
|
|
+ for (String key : ScheduledTasks.allDmdzData.keySet()) {
|
|
|
+ if (splitAddress.getSearchAddr().contains(key)) {
|
|
|
+ dmdzList = ScheduledTasks.allDmdzData.get(key);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for (YyskDmdzAddressStandardization item : dmdzList) {
|
|
|
+ if (item.getCounty().contains(splitAddress.getDistrict())) {
|
|
|
+ if (list.size() < 20) {
|
|
|
+ list.add(item);
|
|
|
+ } else {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+// TODO 数据库查询的方式
|
|
|
+// list = yyszAddressRepository.getAddressPoisByAddrAQX(splitAddress.getDistrict(), splitAddress.getSearchAddr());
|
|
|
}
|
|
|
if (list == null) {
|
|
|
- list = yyszAddressRepository.getAddressPoisByAddr(splitAddress.getSearchAddr());
|
|
|
+ list = getAddressPoiOnlyDB(splitAddress.getSearchAddr());
|
|
|
}
|
|
|
-
|
|
|
return list;
|
|
|
}
|
|
|
|
|
|
- public List<yyskDmdzAddressStandardization> getAddressPoiOnlyDB(String addr) {
|
|
|
- return yyszAddressRepository.getAddressPoisByAddr(addr);
|
|
|
+ public List<YyskDmdzAddressStandardization> getAddressPoiOnlyDB(String addr) {
|
|
|
+// TODO 数据库查询逻辑调整为内存处理的方式
|
|
|
+ if (ScheduledTasks.allDmdzData.containsKey(addr)) {
|
|
|
+ return ScheduledTasks.allDmdzData.get(addr);
|
|
|
+ } else {
|
|
|
+ for (String key : ScheduledTasks.allDmdzData.keySet()) {
|
|
|
+ if (addr.contains(key)) {
|
|
|
+ return ScheduledTasks.allDmdzData.get(key);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+// TODO 数据库查询的方式
|
|
|
+// return yyszAddressRepository.getAddressPoisByAddr(addr);
|
|
|
}
|
|
|
|
|
|
- public List<yyskDmdzAddressStandardization> getRecommendAddress(SplitAddress splitAddress) {
|
|
|
+ public List<YyskDmdzAddressStandardization> getRecommendAddress(SplitAddress splitAddress) {
|
|
|
if (splitAddress.getCity() == null) {
|
|
|
splitAddress.setCity("");
|
|
|
}
|