root/ludu-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

mysql票务抽数新增校验、修改插入方式为批量插入、mongo票务抽数代码优化

Browse Source
This commit is contained in:
XinWei 2024-09-09 11:08:09 +08:00
parent f9982b7ec0
commit 4c3d9c7878
10 changed files with 143 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/ticketing/TicketingController.java
View File

@ -12,7 +12,6 @@ import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
@ -41,42 +40,50 @@ public class TicketingController {
private LargeScreenClient largeScreenClient;
@GetMapping
public CommonResult<String> testTicketing(@RequestParam("startDate") String startDate, @RequestParam("endDate")String endDate, @RequestParam("method") String method) throws Exception {
public CommonResult<String> testTicketing(@RequestParam("startDate") String startDate, @RequestParam("endDate") String endDate, @RequestParam("method") String method) throws Exception {
Calendar startCalendar = Calendar.getInstance();
Calendar endCalendar = Calendar.getInstance();
startCalendar.setTime(TickingDateUtils.parse(startDate));
endCalendar.setTime(TickingDateUtils.parse(endDate));
return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, endCalendar, method));
}
@GetMapping("/test1")
public CommonResult<List<String>> testAll(){
return CommonResult.success(checkTicketService.deleteTwoYearLastMonthAgo());
@GetMapping("/2")
public CommonResult<String> testTicketing2(@RequestParam("method") String method) throws Exception {
Calendar startCalendar = Calendar.getInstance();
startCalendar.add(Calendar.DAY_OF_MONTH, 1);
startCalendar.add(Calendar.YEAR, -1);
return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, startCalendar, method));
}
private void methodUtils(Map<String, Object> map, String name, Object list){
map.put(name,list);
@GetMapping("/intoMysql")
public CommonResult<String> testTicketingMysql(@RequestParam("startDate") String startDate, @RequestParam("endDate") String endDate, @RequestParam("method") String method) throws Exception {
return CommonResult.success("");
}
private String methodDataUtils(){
private void methodUtils(Map<String, Object> map, String name, Object list) {
map.put(name, list);
}
private String methodDataUtils() {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
LocalDateTime now = LocalDateTime.now();
return now.format(formatter);
}
@GetMapping("/test")
public void test(){
public void test() {
Map<String, Object> map = new HashMap<>();
String dataUtils = this.methodDataUtils();
this.methodUtils(map, "assetCount", largeScreenClient.assetMethod1());
this.methodUtils(map,"saleDataTotal",largeScreenClient.saleDataTotal(dataUtils,dataUtils));
this.methodUtils(map,"checkTicketTotal",largeScreenClient.checkTicketTotal(dataUtils,dataUtils));
webSocketSenderApi.sendObject(2,1L,"1",map);
this.methodUtils(map, "saleDataTotal", largeScreenClient.saleDataTotal(dataUtils, dataUtils));
this.methodUtils(map, "checkTicketTotal", largeScreenClient.checkTicketTotal(dataUtils, dataUtils));
webSocketSenderApi.sendObject(2, 1L, "1", map);
}
@GetMapping("/testa")
public CommonResult<List<String>> testAlla(){
public CommonResult<List<String>> testAlla() {
return CommonResult.success(saleDataService.deleteTwoYearLastMonthAgo());
}
@GetMapping("/testSample")
public void testSample(){
ticketDataMigration.deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow();
}
}

7
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/checkticket/CheckTicketMapper.java
View File

@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.infra.controller.checkticket.vo.CheckTicketPageRe
import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketDO;
import com.baomidou.dynamic.datasource.annotation.DS;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
/**
* 检票 Mapper
@ -32,4 +33,10 @@ public interface CheckTicketMapper extends BaseMapperX<CheckTicketDO> {
.orderByDesc(CheckTicketDO::getId));
}
/**
* 根据检票日期物理删除数据
* @param checkticketdate
* @return java.lang.Integer
*/
Integer physicalDeleteByCheckticketdate(@Param("checkticketdate") String checkticketdate);
}

12
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/saledata/SaleDataMapper.java
View File

@ -6,10 +6,7 @@ import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.infra.controller.saledata.vo.SaleDataPageReqVO;
import cn.iocoder.yudao.module.infra.dal.dataobject.saledata.SaleDataDO;
import com.baomidou.dynamic.datasource.annotation.DS;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Result;
import org.apache.ibatis.annotations.Results;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.*;
import java.util.List;
import java.util.Map;
@ -43,4 +40,11 @@ public interface SaleDataMapper extends BaseMapperX<SaleDataDO> {
.likeIfPresent(SaleDataDO::getSalepropetyvaluename, reqVO.getSalepropetyvaluename())
.orderByDesc(SaleDataDO::getId));
}
/**
* 根据销售日期物理删除
* @param sddate
* @return java.lang.Integer
*/
Integer physicalDeleteBySddate(@Param("sddate") String sddate);
}

353
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/job/ticketing/TicketDataMigration.java
View File

@ -59,274 +59,119 @@ public class TicketDataMigration {
* 存入售票数据的定时任务mysql
*/
@XxlJob("getSaleData")
public void getSaleDataHandler() throws Exception {
public String getSaleDataHandler() throws Exception {
int failedCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务：mysql销售数据开始抽数...");
StringJoiner result = new StringJoiner("\n");
XxlJobHelper.log("抽数服务：mysql售票数据开始抽数...");
result.add("抽数服务：mysql售票数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
XxlJobHelper.log("日期：{}暂无数据！", TickingDateUtils.getPreviousDayFormat());
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "暂无数据！");
return result.toString();
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
if (totalRows <= PAGE_SIZE) {
failedCount += insertSaleData(mapList, totalRows, pageNumber);
} else {
// 总条数大于每页条数 执行完第一次插入后根据返回的页数继续进行请求和插入
// 插入第一页的数据
failedCount += insertSaleData(mapList, PAGE_SIZE, pageNumber);
pageNumber++;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理，得到最后一页的实际条数
if (pageNumber != totalPages) {
failedCount += insertSaleData(mapList, PAGE_SIZE, pageNumber);
} else {
failedCount += insertSaleData(mapList, totalRows % PAGE_SIZE, pageNumber);
}
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理，得到最后一页的实际条数
if (!saleDataService.createSaleData(BeanUtils.toBean(mapList, SaleDataDO.class))) {
failedCount += ticketingSamplingRespVO.getPageSize();
}
}
if (failedCount != 0) {
XxlJobHelper.handleFail("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
result.add("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
return result.toString();
}
if (!saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致！！！");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getSaleDataByMongoDBHandler();
XxlJobHelper.log("抽数完，比对结果：{}", saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
XxlJobHelper.log("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
result.add("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
Calendar yesterday = Calendar.getInstance();
yesterday.add(Calendar.DAY_OF_MONTH, -1);
getTicketDataByTimeZonesToMongo(yesterday, yesterday, "saleData");
if (!saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.handleFail("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql售票数据不一致！！！");
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql售票数据不一致！！！");
}
}
XxlJobHelper.log("抽数服务：销售数据抽数正常结束！总共：{}条数据，{}条数据插入失败！", totalRows, failedCount);
XxlJobHelper.log("抽数服务：售票数据抽数正常结束！总共：{}条数据，{}条数据插入失败！", totalRows, failedCount);
result.add("抽数服务：售票数据抽数正常结束！总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！");
return result.toString();
}
/**
* 存入检票数据的定时任务mysql
*/
@XxlJob("getCheckTicket")
public void getCheckTicketHandler() throws Exception {
public String getCheckTicketHandler() throws Exception {
int failedCount = 0;
int pageNumber = 1;
StringJoiner result = new StringJoiner("\n");
result.add("抽数服务：mysql检票数据开始抽数...");
XxlJobHelper.log("抽数服务：mysql检票数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
XxlJobHelper.log("日期：{}暂无数据！", TickingDateUtils.getPreviousDayFormat());
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "暂无数据！");
return result.toString();
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
if (totalRows <= PAGE_SIZE) {
failedCount += insertCheckTicket(mapList, totalRows, pageNumber);
} else {
// 总条数大于每页条数 执行完第一次插入后根据返回的页数继续进行请求和插入
// 插入第一页的数据
failedCount += insertCheckTicket(mapList, PAGE_SIZE, pageNumber);
pageNumber++;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理，得到最后一页的实际条数
if (pageNumber != totalPages) {
failedCount += insertCheckTicket(mapList, PAGE_SIZE, pageNumber);
} else {
failedCount += insertCheckTicket(mapList, totalRows % PAGE_SIZE, pageNumber);
}
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
if (!checkTicketService.createCheckTicket(BeanUtils.toBean(mapList, CheckTicketDO.class))) {
failedCount += ticketingSamplingRespVO.getPageSize();
}
}
if (failedCount != 0) {
XxlJobHelper.handleFail("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
result.add("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
return result.toString();
}
if (!checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致！！！");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getCheckTicketByMongoDBHandler();
XxlJobHelper.log("抽数完，比对结果：{}", checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
XxlJobHelper.log("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
result.add("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
Calendar yesterday = Calendar.getInstance();
yesterday.add(Calendar.DAY_OF_MONTH, -1);
getTicketDataByTimeZonesToMongo(yesterday, yesterday, "checkTicket");
if (!checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.handleFail("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql检票数据不一致！！！");
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql检票数据不一致！！！");
}
}
XxlJobHelper.log("抽数服务：检票数据抽数正常结束！总共：{}条数据，{}条数据插入失败！", totalRows, failedCount);
result.add("抽数服务：检票数据抽数正常结束！总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！");
return result.toString();
}
/**
* 存入销售数据的定时任务mongodb
*/
@XxlJob("getSaleDataByMongoDB")
public void getSaleDataByMongoDBHandler() throws Exception {
int successCount = 0;
int duplicatesCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务：mongodb销售数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = saleDataService.insertTodayByMongoDB(BeanUtils.toBean(mapList, SaleDataToday.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
}
XxlJobHelper.log("抽数服务：检票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", totalRows, duplicatesCount, successCount);
public String getSaleDataByMongoDBHandler() throws Exception {
Calendar now = Calendar.getInstance();
return getTicketDataByTimeZonesToMongo(now, now, "saleData");
}
/**
* 存入检票数据的定时任务mongodb
*/
@XxlJob("getCheckTicketByMongoDB")
public void getCheckTicketByMongoDBHandler() throws Exception {
XxlJobHelper.log("抽数服务：mongodb检票数据开始抽数...");
int successCount = 0;
int duplicatesCount = 0;
int pageNumber = 1;
String date = TickingDateUtils.getNowDayFormat();
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, date, pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, date, pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = checkTicketService.insertTodayByMongoDB(BeanUtils.toBean(mapList, CheckTicketToday.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
}
XxlJobHelper.log("抽数服务：检票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", totalRows, duplicatesCount, successCount);
public String getCheckTicketByMongoDBHandler() throws Exception {
Calendar now = Calendar.getInstance();
return getTicketDataByTimeZonesToMongo(now, now, "checkTicket");
}
@XxlJob("getCheckTicketByMongoDBLastMonthToNow")
public void getCheckTicketByMongoDBLastMonthToNow() throws Exception {
XxlJobHelper.log("抽数服务：mongodb检票数据开始抽数...");
int successCount;
int duplicatesCount;
int pageNumber;
// 获取当前日期
Calendar startDate = Calendar.getInstance();
// 设置开始日期为上一个月的第一天
startDate.set(Calendar.DAY_OF_MONTH, 1); // 先将日期设置为当前月的第一天
startDate.add(Calendar.MONTH, -1); // 然后减去一个月
// 设置结束日期为今天
Calendar endDate = Calendar.getInstance();
// 循环从开始日期到结束日期
while (!startDate.after(endDate)) {
successCount = 0;
duplicatesCount = 0;
String dateString = TickingDateUtils.format(startDate.getTime());
pageNumber = 1;
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, dateString, pageNumber);
if (ticketingSamplingRespVO == null) {
startDate.add(Calendar.DAY_OF_MONTH, 1);
continue;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, dateString, pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = checkTicketService.insertByMongoDB(BeanUtils.toBean(mapList, CheckTicket.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
}
int currentMonth = startDate.get(Calendar.MONTH) + 1; // 加1以得到1-12的月份
int currentDay = startDate.get(Calendar.DAY_OF_MONTH);
XxlJobHelper.log("抽数服务：{}月{}日：检票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", currentMonth, currentDay, totalRows, duplicatesCount, successCount);
// 日期递增一天
startDate.add(Calendar.DAY_OF_MONTH, 1);
}
XxlJobHelper.log("抽数结束!!!");
}
@XxlJob("getSaleDataByMongoDBLastMonthToNow")
public void getSaleDataByMongoDBLastMonthToNow() throws Exception {
XxlJobHelper.log("抽数服务：mongodb售票数据开始抽数...");
int successCount;
int duplicatesCount;
int pageNumber;
// 获取当前日期
Calendar startDate = Calendar.getInstance();
// 设置开始日期为上一个月的第一天
startDate.set(Calendar.DAY_OF_MONTH, 1); // 先将日期设置为当前月的第一天
startDate.add(Calendar.MONTH, -1); // 然后减去一个月
// 设置结束日期为今天
Calendar endDate = Calendar.getInstance();
// 循环从开始日期到结束日期
while (!startDate.after(endDate)) {
successCount = 0;
duplicatesCount = 0;
String dateString = TickingDateUtils.format(startDate.getTime());
pageNumber = 1;
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, dateString, pageNumber);
if (ticketingSamplingRespVO == null) {
startDate.add(Calendar.DAY_OF_MONTH, 1);
continue;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, dateString, pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = saleDataService.insertByMongoDB(BeanUtils.toBean(mapList, SaleData.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
}
int currentMonth = startDate.get(Calendar.MONTH) + 1; // 加1以得到1-12的月份
int currentDay = startDate.get(Calendar.DAY_OF_MONTH);
XxlJobHelper.log("抽数服务：{}月{}日：售票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", currentMonth, currentDay, totalRows, duplicatesCount, successCount);
// 日期递增一天
startDate.add(Calendar.DAY_OF_MONTH, 1);
}
XxlJobHelper.log("抽数结束!!!");
}
/**
* 删除（今年和去年）往前推31天外的数据和抽取去年明天的数据
@ -396,6 +241,10 @@ public class TicketDataMigration {
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(respService, respMethod, dateString, pageNumber);
if (ticketingSamplingRespVO == null) {
startDate.add(Calendar.DAY_OF_MONTH, 1);
result.add("日期：" + TickingDateUtils.format(startDate.getTime()) + "返回无值");
if (startDate.equals(endDate)) { // 如果日期相等，退出循环
break;
}
continue;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
@ -437,48 +286,6 @@ public class TicketDataMigration {
return result.toString();
}
/**
* 插入销售数据的方法（mysql）
*
* @param mapList 数据集合数组
* @param listSize 实际个数
* @param pageNumber 每页显示条数
* @return int 失败条数
*/
private int insertSaleData(List<Map<String, Object>> mapList, int listSize, int pageNumber) {
int failedCount = 0;
for (int i = 0; i < listSize; i++) {
Map<String, Object> currentMap = mapList.get(i);
SaleDataDO saleDataDO = BeanUtils.toBean(currentMap, SaleDataDO.class);
if (saleDataService.createSaleData(saleDataDO) == 0) {
failedCount++;
XxlJobHelper.log("第{}条数据插入失败，数据值为：{}", ((pageNumber - 1) * PAGE_SIZE + i + 1), currentMap);
}
}
return failedCount;
}
/**
* 插入检票数据的方法（mysql）
*
* @param mapList 数据集合数组
* @param listSize 实际个数
* @param pageNumber 每页显示条数
* @return int 失败条数
*/
private int insertCheckTicket(List<Map<String, Object>> mapList, int listSize, int pageNumber) {
int failedCount = 0;
for (int i = 0; i < listSize; i++) {
Map<String, Object> currentMap = mapList.get(i);
CheckTicketDO checkTicketDO = BeanUtils.toBean(currentMap, CheckTicketDO.class);
if (checkTicketService.createCheckTicket(checkTicketDO) == 0) {
failedCount++;
XxlJobHelper.log("第{}条数据插入失败，数据值为：{}", ((pageNumber - 1) * PAGE_SIZE + i + 1), currentMap);
}
}
return failedCount;
}
/**
* 请求目标url的响应数据
*
@ -519,14 +326,4 @@ public class TicketDataMigration {
return ticketingSamplingRespVO;
}
/**
* 校验数据
*
* @param date
* @return java.lang.Boolean
*/
public Boolean dataValidation(String date) {
return checkTicketService.dataValidation(date) && saleDataService.dataValidation(date);
}
}

6
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketService.java
View File

@ -19,12 +19,10 @@ import java.util.Map;
public interface CheckTicketService {
/**
* 创建检票
*
* @param createReqVO 创建信息
* 批量插入检票数据(mysql)
* @return 编号
*/
Long createCheckTicket(@Valid CheckTicketDO createReqVO);
Boolean createCheckTicket(List<CheckTicketDO> checkTicketDOList);
/**
* 查询重复的dataId(mongodb)

7
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketServiceImpl.java
View File

@ -48,11 +48,10 @@ public class CheckTicketServiceImpl implements CheckTicketService {
private MongoTemplate mongoTemplate;
@Override
public Long createCheckTicket(CheckTicketDO checkTicketDO) {
// 插入
checkTicketMapper.insert(checkTicketDO);
public Boolean createCheckTicket(List<CheckTicketDO> checkTicketDOList) {
checkTicketMapper.physicalDeleteByCheckticketdate(checkTicketDOList.get(0).getCheckticketdate());
// 返回
return checkTicketDO.getId();
return checkTicketMapper.insertBatch(checkTicketDOList);
}
@Override

6
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataService.java
View File

@ -21,12 +21,12 @@ import java.util.Map;
public interface SaleDataService {
/**
* 创建售票
* 批量插入售票数据(mysql)
*
* @param saleDataDO 创建信息
* @param saleDataDOList
* @return 编号
*/
Long createSaleData(@Valid SaleDataDO saleDataDO);
Boolean createSaleData(List<SaleDataDO> saleDataDOList);
/**
* 查询重复的dataId(mongodb)

8
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataServiceImpl.java
View File

@ -42,11 +42,9 @@ public class SaleDataServiceImpl implements SaleDataService {
private MongoTemplate mongoTemplate;
@Override
public Long createSaleData(SaleDataDO saleDataDO) {
// 插入
saleDataMapper.insert(saleDataDO);
// 返回
return saleDataDO.getId();
public Boolean createSaleData(List<SaleDataDO> saleDataDOList) {
saleDataMapper.physicalDeleteBySddate(saleDataDOList.get(0).getSddate());
return saleDataMapper.insertBatch(saleDataDOList);
}

9
yudao-module-infra/yudao-module-infra-biz/src/main/resources/mapper/checkticket/CheckticketMapper.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.iocoder.yudao.module.infra.dal.mysql.checkticket.CheckTicketMapper">
<delete id="physicalDeleteByCheckticketdate">
DELETE
FROM checkticket
WHERE checkticketdate = #{checkticketdate}
</delete>
</mapper>

9
yudao-module-infra/yudao-module-infra-biz/src/main/resources/mapper/saledata/SaleDataMapper.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.iocoder.yudao.module.infra.dal.mysql.saledata.SaleDataMapper">
<delete id="physicalDeleteBySddate">
DELETE
FROM saledata
WHERE sddate = #{sddate}
</delete>
</mapper>