Compare commits

...

6 Commits

Author SHA1 Message Date
d6687f97f1 心跳数据推送 2024-09-03 09:49:51 +08:00
0d8fa5efb1 Merge pull request '抽数:验证mysql和mongo数量是否一致' (#68) from xinwei into master
All checks were successful
continuous-integration/drone Build is passing
Reviewed-on: #68
2024-09-02 11:00:26 +08:00
3eca0681d9 抽数:验证mysql和mongo数量是否一致 2024-09-02 10:58:13 +08:00
0f7f5797f8 Merge pull request '构建文件' (#67) from xinwei into master
All checks were successful
continuous-integration/drone Build is passing
Reviewed-on: #67
2024-08-30 17:38:09 +08:00
ad0ff4b78e 构建文件 2024-08-30 17:37:24 +08:00
c87a34dd3a Merge pull request '心跳数据推送' (#66) from zzw-one into master
All checks were successful
continuous-integration/drone/tag Build is passing
Reviewed-on: #66
2024-08-30 17:27:03 +08:00
14 changed files with 299 additions and 32 deletions

39
.drone-gateway.yml Normal file
View File

@ -0,0 +1,39 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-gateway-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: gateway-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/yudao-gateway
- chmod +x run.sh
- ./run.sh || echo "gateway build failed"

39
.drone-infra.yml Normal file
View File

@ -0,0 +1,39 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-infra-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: infra-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/yudao-module-infra/yudao-module-infra-biz
- chmod +x run.sh
- ./run.sh || echo "infra build failed"

39
.drone-job.yml Normal file
View File

@ -0,0 +1,39 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-job-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: job-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/ludu-job-admin/ludu-job-admin-biz
- chmod +x run.sh
- ./run.sh || echo "job build failed"

39
.drone-system.yml Normal file
View File

@ -0,0 +1,39 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-system-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: system-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/yudao-module-system/yudao-module-system-biz
- chmod +x run.sh
- ./run.sh || echo "system build failed"

View File

@ -2,6 +2,7 @@ package cn.iocoder.yudao.module.infra.clients;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.List;
import java.util.Map;
@ -9,5 +10,11 @@ import java.util.Map;
@FeignClient("datacenter-server")
public interface LargeScreenClient {
@GetMapping("/api/asset/type")
List<Object> assetMethod1();
Object assetMethod1();
@GetMapping("/api/saledata")
Object saleDataTotal(@RequestParam("starTime") String starTime, @RequestParam("endTime") String endTime);
@GetMapping("/api/checkticket")
Object checkTicketTotal(@RequestParam("starTime") String starTime, @RequestParam("endTime") String endTime);
}

View File

@ -2,6 +2,8 @@ package cn.iocoder.yudao.module.infra.controller.admin.ticketing;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
import cn.iocoder.yudao.module.infra.api.websocket.WebSocketSenderApi;
import cn.iocoder.yudao.module.infra.clients.LargeScreenClient;
import cn.iocoder.yudao.module.infra.job.ticketing.TicketDataMigration;
import cn.iocoder.yudao.module.infra.service.checkticket.CheckTicketService;
import cn.iocoder.yudao.module.infra.service.saledata.SaleDataService;
@ -10,9 +12,10 @@ import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
/**
* @Description 测试抽数接口
@ -31,6 +34,12 @@ public class TicketingController {
@Resource
SaleDataService saleDataService;
@Resource
private WebSocketSenderApi webSocketSenderApi;
@Resource
private LargeScreenClient largeScreenClient;
@GetMapping
public CommonResult<String> testTicketing(@RequestParam("startDate") String startDate, @RequestParam("endDate")String endDate, @RequestParam("method") String method) throws Exception {
Calendar startCalendar = Calendar.getInstance();
@ -39,10 +48,29 @@ public class TicketingController {
endCalendar.setTime(TickingDateUtils.parse(endDate));
return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, endCalendar, method));
}
@GetMapping("/test")
@GetMapping("/test1")
public CommonResult<List<String>> testAll(){
return CommonResult.success(checkTicketService.deleteTwoYearLastMonthAgo());
}
private void methodUtils(Map<String, Object> map, String name, Object list){
map.put(name,list);
}
private String methodDataUtils(){
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
LocalDateTime now = LocalDateTime.now();
return now.format(formatter);
}
@GetMapping("/test")
public void test(){
Map<String, Object> map = new HashMap<>();
String dataUtils = this.methodDataUtils();
this.methodUtils(map, "assetCount", largeScreenClient.assetMethod1());
this.methodUtils(map,"saleDataTotal",largeScreenClient.saleDataTotal(dataUtils,dataUtils));
this.methodUtils(map,"checkTicketTotal",largeScreenClient.checkTicketTotal(dataUtils,dataUtils));
webSocketSenderApi.sendObject(2,1L,"1",map);
}
@GetMapping("/testa")
public CommonResult<List<String>> testAlla(){
return CommonResult.success(saleDataService.deleteTwoYearLastMonthAgo());

View File

@ -12,24 +12,29 @@ import java.util.List;
@Repository
public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketToday,String> {
public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketToday, String> {
/**
* 根据dataId数组查找
*
* @param dataIdList
* @return java.util.List<cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket>
*/
@Query(value = "{ 'dataId': { $in: ?0 } }", fields = "{ 'dataId': 1, '_id': 0 }")
List<CheckTicketToday> findByDataIdIn(List<String> dataIdList);
/**
* 找到所有不在时间段内的日期字段
*
* @param startDate
* @param endDate
* @return java.util.List<cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketToday>
*/
@Query("{ 'checkticketdate': { $not: { $gte: ?0, $lt: ?1 } } }")
List<CheckTicketToday> findNotInTimeRange(String startDate, String endDate);
/**
* 查询所有去重的 checkticketdate 字段值
*
* @return 去重后的 checkticketdate 字段值列表
*/
@Aggregation(pipeline = {
@ -39,9 +44,10 @@ public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketT
List<String> findDistinctDate();
/**
* 删除 checkticketdate 字段值在给定列表中的所有记录
* @param dates 日期列表
* 计算自定义日期的总数
* @param date
* @return int
*/
@Query("{ 'checkticketdate': { $in: ?0 } }")
void deleteByCheckticketdateIn(List<String> dates);
@Query(value = "{ 'checkticketdate': ?0 }", count = true)
long countByCustomDate(String date);
}

View File

@ -87,4 +87,12 @@ public interface SaleDataTodayRepository extends MongoRepository<SaleDataToday,S
"{$project: {_id: 0, sddate: '$_id'}}"
})
List<String> findDistinctDate();
/**
* 根据日期计算总数
* @param date
* @return java.lang.Long
*/
@Query(value = "{ 'sddate': ?0 }", count = true)
Long countByCustomDate(String date);
}

View File

@ -7,6 +7,9 @@ import com.xxl.job.core.handler.annotation.XxlJob;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -19,10 +22,25 @@ public class LargeScreenHeartbeat {
@Resource
private LargeScreenClient largeScreenClient;
private void methodUtils(Map<String, Object> map, String name, Object list){
map.put(name,list);
}
private String methodDataUtils(){
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
LocalDateTime now = LocalDateTime.now();
return now.format(formatter);
}
@XxlJob("largeScreen")
public void largeScreenToWebSocket(){
List<Object> maps = largeScreenClient.assetMethod1();
webSocketSenderApi.sendObject(1,"1",maps);
Map<String, Object> map = new HashMap<>();
String dataUtils = this.methodDataUtils();
this.methodUtils(map, "assetCount", largeScreenClient.assetMethod1());
this.methodUtils(map,"saleDataTotal",largeScreenClient.saleDataTotal(dataUtils,dataUtils));
this.methodUtils(map,"checkTicketTotal",largeScreenClient.checkTicketTotal(dataUtils,dataUtils));
webSocketSenderApi.sendObject(2,1L,"1",map);
}
}

View File

@ -63,7 +63,11 @@ public class TicketDataMigration {
int failedCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务mysql销售数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
@ -78,7 +82,7 @@ public class TicketDataMigration {
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理得到最后一页的实际条数
@ -90,7 +94,12 @@ public class TicketDataMigration {
}
}
if (!saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getSaleDataByMongoDBHandler();
XxlJobHelper.log("抽数完,比对结果:{}", saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
}
XxlJobHelper.log("抽数服务:销售数据抽数正常结束!总共:{}条数据,{}条数据插入失败!", totalRows, failedCount);
}
@ -102,7 +111,11 @@ public class TicketDataMigration {
int failedCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务mysql检票数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
@ -117,7 +130,7 @@ public class TicketDataMigration {
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理得到最后一页的实际条数
if (pageNumber != totalPages) {
@ -126,9 +139,13 @@ public class TicketDataMigration {
failedCount += insertCheckTicket(mapList, totalRows % PAGE_SIZE, pageNumber);
}
}
}
if (!checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getCheckTicketByMongoDBHandler();
XxlJobHelper.log("抽数完,比对结果:{}", checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
}
XxlJobHelper.log("抽数服务:检票数据抽数正常结束!总共:{}条数据,{}条数据插入失败!", totalRows, failedCount);
}
@ -142,6 +159,10 @@ public class TicketDataMigration {
int pageNumber = 1;
XxlJobHelper.log("抽数服务mongodb销售数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
@ -173,6 +194,10 @@ public class TicketDataMigration {
int pageNumber = 1;
String date = TickingDateUtils.getNowDayFormat();
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, date, pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
@ -307,7 +332,7 @@ public class TicketDataMigration {
* 删除今年和去年往前推31天外的数据和抽取去年明天的数据
*/
@XxlJob("deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow")
public void deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow(){
public void deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow() {
XxlJobHelper.log("删除多余数据开始...");
// 检票数据今年去年多余删除
XxlJobHelper.log("删除检票日期为{}", checkTicketService.deleteTwoYearLastMonthAgo().toString());
@ -401,7 +426,7 @@ public class TicketDataMigration {
int currentDay = startDate.get(Calendar.DAY_OF_MONTH);
result.add(String.format("抽数服务:%d年%d月%d日" + methodName + "数据抽数正常结束!总共:%d条数据%d条数据重复%d条数据插入成功", currentYear, currentMonth, currentDay, totalRows, duplicatesCount, successCount));
XxlJobHelper.log("抽数服务:{}年{}月{}日:{}数据抽数正常结束!总共:{}条数据,{}条数据重复,{}条数据插入成功!", currentYear, currentMonth, currentDay, methodName, totalRows, duplicatesCount, successCount);
if (startDate.equals(endDate)){ // 如果日期相等退出循环
if (startDate.equals(endDate)) { // 如果日期相等退出循环
break;
}
// 日期递增一天
@ -488,25 +513,20 @@ public class TicketDataMigration {
XxlJobHelper.handleFail("数据抽数失败,请重试");
throw new RuntimeException();
} else if (ticketingSamplingRespVO.getPageNumber() == null || ticketingSamplingRespVO.getPageSize() == null || ticketingSamplingRespVO.getTotalPages() == null || ticketingSamplingRespVO.getTotalRows() == null || ticketingSamplingRespVO.getDataMapList().isEmpty()) {
XxlJobHelper.log("{},请求抽数成功,但返回参数存在问题{}", date, ticketingSamplingRespVO);
XxlJobHelper.log("{},请求抽数成功,但返回参数值:{}", date, ticketingSamplingRespVO);
return null;
}
return ticketingSamplingRespVO;
}
/**
* 请求失败消息
* 校验数据
*
* @param
* @return void
* @param date
* @return java.lang.Boolean
*/
private void requestFailedMessage(TicketingSamplingRespVO ticketingSamplingRespVO) {
// 如果访问失败则让调度中心重新调用
if (!ticketingSamplingRespVO.isPassflag()) {
// 让调度中心重新调用
XxlJobHelper.log("抽数服务:检票数据抽数失败,返回消息:" + ticketingSamplingRespVO.getRtnMsg());
XxlJobHelper.handleFail("销售数据抽数失败,请重试");
}
public Boolean dataValidation(String date) {
return checkTicketService.dataValidation(date) && saleDataService.dataValidation(date);
}
}

View File

@ -56,4 +56,11 @@ public interface CheckTicketService {
List<String> deleteTwoYearLastMonthAgo();
List<String> findDistinctDate();
/**
* mongo和mysql数据数量验证
* @param date
* @return java.lang.Boolean
*/
Boolean dataValidation(String date);
}

View File

@ -149,4 +149,9 @@ public class CheckTicketServiceImpl implements CheckTicketService {
return checkTicketTodayRepository.findDistinctDate();
}
@Override
public Boolean dataValidation(String date) {
return checkTicketMapper.selectCount(CheckTicketDO::getCheckticketdate, date) == checkTicketTodayRepository.countByCustomDate(date);
}
}

View File

@ -55,4 +55,11 @@ public interface SaleDataService {
* @return java.util.List<java.lang.String>
*/
List<String> deleteTwoYearLastMonthAgo();
/**
* mongo和mysql数据数量验证
* @param date
* @return boolean
*/
Boolean dataValidation(String date);
}

View File

@ -136,4 +136,9 @@ public class SaleDataServiceImpl implements SaleDataService {
mongoTemplate.remove(query, "sale_data_today");
return filteredDate;
}
@Override
public Boolean dataValidation(String date) {
return Objects.equals(saleDataMapper.selectCount(SaleDataDO::getSddate, date), saleDataTodayRepository.countByCustomDate(date));
}
}