|
@@ -27,10 +27,9 @@ public class TestDataServiceImpl {
|
|
|
private TestDataRepository testDataRepository;
|
|
|
|
|
|
// 分页查询地址表
|
|
|
- public Page<TestData> getAllDataPage(int page, int size) {
|
|
|
- Pageable pageable = PageRequest.of(page - 1, size);
|
|
|
-// return testDataRepository.findByAddress1Containing(Constant.getArea(), pageable);
|
|
|
- return testDataRepository.findAll(pageable);
|
|
|
+ public Page<TestData> getAllDataPage(int size) {
|
|
|
+ Pageable pageable = PageRequest.of(0, size);
|
|
|
+ return testDataRepository.findByAddress1Containing(pageable);
|
|
|
}
|
|
|
|
|
|
// 批量修改地址表
|
|
@@ -42,114 +41,126 @@ public class TestDataServiceImpl {
|
|
|
/**
|
|
|
* 首先要得到一个Page<T>对象,然后判断是否还有别的数据,有的话接着请求并返回新的Page<T>对象并迭代处理数据
|
|
|
*/
|
|
|
- public void iterativeProcessing(Integer pageNum, Integer pageSize, Integer level, Integer init) {
|
|
|
- System.out.println("<<<<<<<<------iterativeProcessing{pageNum:" + pageNum + ",pageSize:" + pageSize + "}");
|
|
|
- Page<TestData> outData = getAllDataPage(pageNum, pageSize);
|
|
|
- List<TestData> listData = outData.getContent();
|
|
|
- // 创建线程池
|
|
|
- int threadCount = Runtime.getRuntime().availableProcessors();
|
|
|
- ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
|
|
|
- List<Future<?>> futures = new ArrayList<>();
|
|
|
- for (TestData item : listData) {
|
|
|
- futures.add(executorService.submit(() -> {
|
|
|
- if (init == 1) {
|
|
|
- item.setInTheArea(null);
|
|
|
- item.setTwon(null);
|
|
|
- item.setLevel(null);
|
|
|
- item.setAdministrativeDivision(null);
|
|
|
- item.setLat(null);
|
|
|
- item.setLon(null);
|
|
|
- item.setResultAddrKey(null);
|
|
|
- item.setUpdateTime(null);
|
|
|
- } else {
|
|
|
- List<String> addrList = new ArrayList<>();
|
|
|
- if (item.getSkHjdz() != null && !item.getSkHjdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getSkHjdz())) {
|
|
|
-// 实口户籍地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkHjdz()));
|
|
|
- }
|
|
|
- if (item.getSkJzdz() != null && !item.getSkJzdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getSkJzdz())) {
|
|
|
-// 实口居住地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkJzdz()));
|
|
|
- }
|
|
|
- if (item.getJcHjdz() != null && !item.getJcHjdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getJcHjdz())) {
|
|
|
-// 基层户籍地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcHjdz()));
|
|
|
- }
|
|
|
- if (item.getJcJzdz() != null && !item.getJcJzdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getJcJzdz())) {
|
|
|
-// 基层居住地址
|
|
|
- addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcJzdz()));
|
|
|
- }
|
|
|
-// 判断地址是否属于松江区
|
|
|
- if (addrList.size() > 0) {
|
|
|
-// 开始查询
|
|
|
- AddressResult addressResult = AddressQueryEngine.getInstance().commonSearchByName(addrList, AddressLevel.values()[level - 1]);
|
|
|
- if (addressResult != null) {
|
|
|
- if (addressResult.getData() == null || addressResult.getData().size() < 1) {
|
|
|
- item.setInTheArea("异常");
|
|
|
- item.setResultAddrKey("未匹配到符合规则的结果!");
|
|
|
- } else {
|
|
|
- try {
|
|
|
- for (AddressResult.ContentBean contentBean : addressResult.getData()) {
|
|
|
- String resultAddrKey = contentBean.getAddress();
|
|
|
- if (resultAddrKey != null && contentBean.getLon() != null && contentBean.getLat() != null && ExcelReaderUtils.isOtherDistrictThanSongJiang(resultAddrKey)) {
|
|
|
- String lng = contentBean.getLat() + "";
|
|
|
- String lat = contentBean.getLon() + "";
|
|
|
- item.setInTheArea("是");
|
|
|
- if (contentBean.getAdJson() != null) {
|
|
|
+ public void iterativeProcessing(Integer pageSize, Integer level, Integer init) {
|
|
|
+ System.out.println("<<<<<<<<------iterativeProcessing{pageSize:" + pageSize + "}");
|
|
|
+ long startTime = System.currentTimeMillis();
|
|
|
+ Page<TestData> outData = getAllDataPage(pageSize);
|
|
|
+ if (outData.getTotalElements() > 0) {
|
|
|
+ List<TestData> listData = outData.getContent();
|
|
|
+ // 创建线程池
|
|
|
+ ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
|
|
|
+ List<Future<?>> futures = new ArrayList<>();
|
|
|
+ for (TestData item : listData) {
|
|
|
+ futures.add(executorService.submit(() -> {
|
|
|
+ if (init == 1) {
|
|
|
+ item.setInTheArea(null);
|
|
|
+ item.setTwon(null);
|
|
|
+ item.setLevel(null);
|
|
|
+ item.setAdministrativeDivision(null);
|
|
|
+ item.setStandardizedAddress(null);
|
|
|
+ item.setMarket(null);
|
|
|
+ item.setLat(null);
|
|
|
+ item.setLon(null);
|
|
|
+ item.setResultAddrKey(null);
|
|
|
+ item.setUpdateTime(null);
|
|
|
+ } else {
|
|
|
+ List<String> addrList = new ArrayList<>();
|
|
|
+ if (item.getSkHjdz() != null && !item.getSkHjdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getSkHjdz())) {
|
|
|
+// 实口户籍地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkHjdz()));
|
|
|
+ }
|
|
|
+ if (item.getSkJzdz() != null && !item.getSkJzdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getSkJzdz())) {
|
|
|
+// 实口居住地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getSkJzdz()));
|
|
|
+ }
|
|
|
+ if (item.getJcHjdz() != null && !item.getJcHjdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getJcHjdz())) {
|
|
|
+// 基层户籍地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcHjdz()));
|
|
|
+ }
|
|
|
+ if (item.getJcJzdz() != null && !item.getJcJzdz().isEmpty() && ExcelReaderUtils.isOtherDistrictThanSongJiang(item.getJcJzdz())) {
|
|
|
+// 基层居住地址
|
|
|
+ addrList.add("上海市" + Constant.getArea() + "区" + AddressQueryEngine.addressReplaceAll(item.getJcJzdz()));
|
|
|
+ }
|
|
|
+// 判断地址是否属于松江区
|
|
|
+ if (addrList.size() > 0) {
|
|
|
+// 开始查询
|
|
|
+ AddressResult addressResult = AddressQueryEngine.getInstance().commonSearchByName(addrList, AddressLevel.values()[level - 1]);
|
|
|
+ if (addressResult != null) {
|
|
|
+ if (addressResult.getData() == null || addressResult.getData().size() < 1) {
|
|
|
+ item.setInTheArea("异常");
|
|
|
+ item.setResultAddrKey("未匹配到符合规则的结果!");
|
|
|
+ } else {
|
|
|
+ try {
|
|
|
+ for (AddressResult.ContentBean contentBean : addressResult.getData()) {
|
|
|
+ String resultAddrKey = contentBean.getAddress();
|
|
|
+ if (resultAddrKey != null && contentBean.getLon() != null && contentBean.getLat() != null && ExcelReaderUtils.isOtherDistrictThanSongJiang(resultAddrKey) && contentBean.getAdJson() != null && contentBean.getTownJson() != null) {
|
|
|
+ String lng = contentBean.getLat() + "";
|
|
|
+ String lat = contentBean.getLon() + "";
|
|
|
+ item.setInTheArea("是");
|
|
|
if (contentBean.getAdJson().getString("name") != null) {
|
|
|
item.setAdministrativeDivision(contentBean.getAdJson().getString("name"));
|
|
|
}
|
|
|
- }
|
|
|
- if (contentBean.getTownJson() != null) {
|
|
|
if (contentBean.getTownJson().getString("name") != null) {
|
|
|
item.setTwon(contentBean.getTownJson().getString("name"));
|
|
|
}
|
|
|
+ item.setLat(lat);
|
|
|
+ item.setLon(lng);
|
|
|
+ item.setResultAddrKey(resultAddrKey);
|
|
|
+ item.setMarket("上海市");
|
|
|
+// 开始规范化地址(把查询地址的市、区、街镇替换为空,然后把得到的区、街镇拼接上去)(得到区、街镇)
|
|
|
+ String oldAddress = contentBean.getSearchAddress().replaceAll("上海市", "").replaceAll("松江区", "");
|
|
|
+ String[] towns = Constant.getTowns();
|
|
|
+ for (String town : towns) {
|
|
|
+ oldAddress = oldAddress.replaceAll(town, "");
|
|
|
+ }
|
|
|
+ if (contentBean.getCjJson() != null && contentBean.getCjJson().containsKey("所属居委")) {
|
|
|
+ oldAddress = contentBean.getCjJson().getString("所属居委") + oldAddress;
|
|
|
+ }
|
|
|
+ item.setStandardizedAddress("上海市" + item.getAdministrativeDivision() + item.getTwon() + oldAddress);
|
|
|
+ break;
|
|
|
+ } else {
|
|
|
+ item.setInTheArea("异常");
|
|
|
+ item.setResultAddrKey("未匹配到符合规则的结果!");
|
|
|
}
|
|
|
-
|
|
|
- item.setLat(lat);
|
|
|
- item.setLon(lng);
|
|
|
- item.setResultAddrKey(resultAddrKey);
|
|
|
- break;
|
|
|
- } else {
|
|
|
- item.setInTheArea("异常");
|
|
|
- item.setResultAddrKey("未查询到符合条件的结果!");
|
|
|
}
|
|
|
+ } catch (Exception e) {
|
|
|
+ item.setInTheArea("异常");
|
|
|
+ System.err.println("查询结果处理异常:" + e);
|
|
|
}
|
|
|
- } catch (Exception e) {
|
|
|
- item.setInTheArea("异常");
|
|
|
- System.err.println("查询结果处理异常:" + e);
|
|
|
}
|
|
|
+ } else {
|
|
|
+ item.setInTheArea("异常");
|
|
|
+ item.setResultAddrKey("未匹配到符合规则的结果!");
|
|
|
}
|
|
|
} else {
|
|
|
- item.setInTheArea("异常");
|
|
|
- item.setResultAddrKey("未查询到符合条件的结果!");
|
|
|
+// 更新时间,添加是否松江判断字段为否
|
|
|
+ item.setInTheArea("否");
|
|
|
}
|
|
|
- } else {
|
|
|
-// 更新时间,添加是否松江判断字段为否
|
|
|
- item.setInTheArea("否");
|
|
|
+ item.setUpdateTime(new Date());
|
|
|
+ item.setLevel(level.toString());
|
|
|
}
|
|
|
- item.setUpdateTime(new Date());
|
|
|
- item.setLevel(level.toString());
|
|
|
- }
|
|
|
- }));
|
|
|
- }
|
|
|
+ }));
|
|
|
+ }
|
|
|
|
|
|
- // 等待所有任务完成
|
|
|
- for (Future<?> future : futures) {
|
|
|
- try {
|
|
|
- if (future != null) {
|
|
|
+ // 等待所有任务完成
|
|
|
+ for (Future<?> future : futures) {
|
|
|
+ try {
|
|
|
future.get();
|
|
|
+ } catch (InterruptedException | ExecutionException e) {
|
|
|
+ System.err.println("线程异常:" + e);
|
|
|
}
|
|
|
- } catch (InterruptedException | ExecutionException e) {
|
|
|
- System.err.println("线程异常:" + e);
|
|
|
}
|
|
|
- }
|
|
|
- // 关闭线程池
|
|
|
- executorService.shutdown();
|
|
|
- updateDatas(listData);
|
|
|
-// 多线程处理完成后,判断是否有其他分页并迭代
|
|
|
- if ((long) pageNum * pageSize < outData.getTotalElements()) {
|
|
|
- iterativeProcessing(pageNum + 1, pageSize, level, init);
|
|
|
+ // 关闭线程池
|
|
|
+ executorService.shutdown();
|
|
|
+ updateDatas(listData);
|
|
|
+ long endTime = System.currentTimeMillis();
|
|
|
+ System.out.println("处理单批次用时" + (endTime - startTime) / 1000 + "秒!");
|
|
|
+ if (outData.getTotalElements() - pageSize > 0) {
|
|
|
+ System.out.println("剩余处理条数:" + (outData.getTotalElements() - pageSize));
|
|
|
+ iterativeProcessing(pageSize, level, init);
|
|
|
+ } else {
|
|
|
+ System.out.println("<<<<<<<<------任务处理完成!");
|
|
|
+ }
|
|
|
} else {
|
|
|
System.out.println("<<<<<<<<------任务处理完成!");
|
|
|
}
|