Compare commits

..

No commits in common. "d6687f97f122a3c192ec6ec15a18ceab4fa63c34" and "5b1b39418ecdb1a035ac054393a863eda8c0ef5b" have entirely different histories.

14 changed files with 32 additions and 299 deletions

View File

@ -1,39 +0,0 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-gateway-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: gateway-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/yudao-gateway
- chmod +x run.sh
- ./run.sh || echo "gateway build failed"

View File

@ -1,39 +0,0 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-infra-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: infra-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/yudao-module-infra/yudao-module-infra-biz
- chmod +x run.sh
- ./run.sh || echo "infra build failed"

View File

@ -1,39 +0,0 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-job-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: job-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/ludu-job-admin/ludu-job-admin-biz
- chmod +x run.sh
- ./run.sh || echo "job build failed"

View File

@ -1,39 +0,0 @@
kind: pipeline # 定义对象类型还有secret和signature两种类型
type: docker # 定义流水线类型还有kubernetes、exec、ssh等类型
name: ludu-system-drone # 定义流水线名称
clone:
disable: true
steps: # 定义流水线执行步骤,这些步骤将顺序执行
- name: system-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /root/ludu-cloud
- git fetch origin
- git reset --hard origin/master
- git clean -fd
- /root/apache-maven-3.9.4/bin/mvn package -Dmaven.test.skip=true
- cd /root/ludu-cloud/yudao-module-system/yudao-module-system-biz
- chmod +x run.sh
- ./run.sh || echo "system build failed"

View File

@ -2,7 +2,6 @@ package cn.iocoder.yudao.module.infra.clients;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.List;
import java.util.Map;
@ -10,11 +9,5 @@ import java.util.Map;
@FeignClient("datacenter-server")
public interface LargeScreenClient {
@GetMapping("/api/asset/type")
Object assetMethod1();
@GetMapping("/api/saledata")
Object saleDataTotal(@RequestParam("starTime") String starTime, @RequestParam("endTime") String endTime);
@GetMapping("/api/checkticket")
Object checkTicketTotal(@RequestParam("starTime") String starTime, @RequestParam("endTime") String endTime);
List<Object> assetMethod1();
}

View File

@ -2,8 +2,6 @@ package cn.iocoder.yudao.module.infra.controller.admin.ticketing;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
import cn.iocoder.yudao.module.infra.api.websocket.WebSocketSenderApi;
import cn.iocoder.yudao.module.infra.clients.LargeScreenClient;
import cn.iocoder.yudao.module.infra.job.ticketing.TicketDataMigration;
import cn.iocoder.yudao.module.infra.service.checkticket.CheckTicketService;
import cn.iocoder.yudao.module.infra.service.saledata.SaleDataService;
@ -12,10 +10,9 @@ import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
/**
* @Description 测试抽数接口
@ -34,12 +31,6 @@ public class TicketingController {
@Resource
SaleDataService saleDataService;
@Resource
private WebSocketSenderApi webSocketSenderApi;
@Resource
private LargeScreenClient largeScreenClient;
@GetMapping
public CommonResult<String> testTicketing(@RequestParam("startDate") String startDate, @RequestParam("endDate")String endDate, @RequestParam("method") String method) throws Exception {
Calendar startCalendar = Calendar.getInstance();
@ -48,29 +39,10 @@ public class TicketingController {
endCalendar.setTime(TickingDateUtils.parse(endDate));
return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, endCalendar, method));
}
@GetMapping("/test1")
@GetMapping("/test")
public CommonResult<List<String>> testAll(){
return CommonResult.success(checkTicketService.deleteTwoYearLastMonthAgo());
}
private void methodUtils(Map<String, Object> map, String name, Object list){
map.put(name,list);
}
private String methodDataUtils(){
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
LocalDateTime now = LocalDateTime.now();
return now.format(formatter);
}
@GetMapping("/test")
public void test(){
Map<String, Object> map = new HashMap<>();
String dataUtils = this.methodDataUtils();
this.methodUtils(map, "assetCount", largeScreenClient.assetMethod1());
this.methodUtils(map,"saleDataTotal",largeScreenClient.saleDataTotal(dataUtils,dataUtils));
this.methodUtils(map,"checkTicketTotal",largeScreenClient.checkTicketTotal(dataUtils,dataUtils));
webSocketSenderApi.sendObject(2,1L,"1",map);
}
@GetMapping("/testa")
public CommonResult<List<String>> testAlla(){
return CommonResult.success(saleDataService.deleteTwoYearLastMonthAgo());

View File

@ -12,29 +12,24 @@ import java.util.List;
@Repository
public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketToday, String> {
public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketToday,String> {
/**
* 根据dataId数组查找
*
* @param dataIdList
* @return java.util.List<cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket>
*/
@Query(value = "{ 'dataId': { $in: ?0 } }", fields = "{ 'dataId': 1, '_id': 0 }")
List<CheckTicketToday> findByDataIdIn(List<String> dataIdList);
/**
* 找到所有不在时间段内的日期字段
*
* @param startDate
* @param endDate
* @return java.util.List<cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketToday>
*/
@Query("{ 'checkticketdate': { $not: { $gte: ?0, $lt: ?1 } } }")
List<CheckTicketToday> findNotInTimeRange(String startDate, String endDate);
/**
* 查询所有去重的 checkticketdate 字段值
*
* @return 去重后的 checkticketdate 字段值列表
*/
@Aggregation(pipeline = {
@ -44,10 +39,9 @@ public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketT
List<String> findDistinctDate();
/**
* 计算自定义日期的总数
* @param date
* @return int
* 删除 checkticketdate 字段值在给定列表中的所有记录
* @param dates 日期列表
*/
@Query(value = "{ 'checkticketdate': ?0 }", count = true)
long countByCustomDate(String date);
@Query("{ 'checkticketdate': { $in: ?0 } }")
void deleteByCheckticketdateIn(List<String> dates);
}

View File

@ -87,12 +87,4 @@ public interface SaleDataTodayRepository extends MongoRepository<SaleDataToday,S
"{$project: {_id: 0, sddate: '$_id'}}"
})
List<String> findDistinctDate();
/**
* 根据日期计算总数
* @param date
* @return java.lang.Long
*/
@Query(value = "{ 'sddate': ?0 }", count = true)
Long countByCustomDate(String date);
}

View File

@ -7,9 +7,6 @@ import com.xxl.job.core.handler.annotation.XxlJob;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -22,25 +19,10 @@ public class LargeScreenHeartbeat {
@Resource
private LargeScreenClient largeScreenClient;
private void methodUtils(Map<String, Object> map, String name, Object list){
map.put(name,list);
}
private String methodDataUtils(){
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
LocalDateTime now = LocalDateTime.now();
return now.format(formatter);
}
@XxlJob("largeScreen")
public void largeScreenToWebSocket(){
Map<String, Object> map = new HashMap<>();
String dataUtils = this.methodDataUtils();
this.methodUtils(map, "assetCount", largeScreenClient.assetMethod1());
this.methodUtils(map,"saleDataTotal",largeScreenClient.saleDataTotal(dataUtils,dataUtils));
this.methodUtils(map,"checkTicketTotal",largeScreenClient.checkTicketTotal(dataUtils,dataUtils));
webSocketSenderApi.sendObject(2,1L,"1",map);
List<Object> maps = largeScreenClient.assetMethod1();
webSocketSenderApi.sendObject(1,"1",maps);
}
}

View File

@ -63,11 +63,7 @@ public class TicketDataMigration {
int failedCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务mysql销售数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
@ -82,7 +78,7 @@ public class TicketDataMigration {
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理得到最后一页的实际条数
@ -94,12 +90,7 @@ public class TicketDataMigration {
}
}
if (!saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getSaleDataByMongoDBHandler();
XxlJobHelper.log("抽数完,比对结果:{}", saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
}
XxlJobHelper.log("抽数服务:销售数据抽数正常结束!总共:{}条数据,{}条数据插入失败!", totalRows, failedCount);
}
@ -111,11 +102,7 @@ public class TicketDataMigration {
int failedCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务mysql检票数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
@ -130,7 +117,7 @@ public class TicketDataMigration {
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理得到最后一页的实际条数
if (pageNumber != totalPages) {
@ -139,13 +126,9 @@ public class TicketDataMigration {
failedCount += insertCheckTicket(mapList, totalRows % PAGE_SIZE, pageNumber);
}
}
}
if (!checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getCheckTicketByMongoDBHandler();
XxlJobHelper.log("抽数完,比对结果:{}", checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
}
XxlJobHelper.log("抽数服务:检票数据抽数正常结束!总共:{}条数据,{}条数据插入失败!", totalRows, failedCount);
}
@ -159,10 +142,6 @@ public class TicketDataMigration {
int pageNumber = 1;
XxlJobHelper.log("抽数服务mongodb销售数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
@ -194,10 +173,6 @@ public class TicketDataMigration {
int pageNumber = 1;
String date = TickingDateUtils.getNowDayFormat();
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, date, pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据!");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
@ -332,7 +307,7 @@ public class TicketDataMigration {
* 删除今年和去年往前推31天外的数据和抽取去年明天的数据
*/
@XxlJob("deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow")
public void deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow() {
public void deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow(){
XxlJobHelper.log("删除多余数据开始...");
// 检票数据今年去年多余删除
XxlJobHelper.log("删除检票日期为{}", checkTicketService.deleteTwoYearLastMonthAgo().toString());
@ -426,7 +401,7 @@ public class TicketDataMigration {
int currentDay = startDate.get(Calendar.DAY_OF_MONTH);
result.add(String.format("抽数服务:%d年%d月%d日" + methodName + "数据抽数正常结束!总共:%d条数据%d条数据重复%d条数据插入成功", currentYear, currentMonth, currentDay, totalRows, duplicatesCount, successCount));
XxlJobHelper.log("抽数服务:{}年{}月{}日:{}数据抽数正常结束!总共:{}条数据,{}条数据重复,{}条数据插入成功!", currentYear, currentMonth, currentDay, methodName, totalRows, duplicatesCount, successCount);
if (startDate.equals(endDate)) { // 如果日期相等退出循环
if (startDate.equals(endDate)){ // 如果日期相等退出循环
break;
}
// 日期递增一天
@ -513,20 +488,25 @@ public class TicketDataMigration {
XxlJobHelper.handleFail("数据抽数失败,请重试");
throw new RuntimeException();
} else if (ticketingSamplingRespVO.getPageNumber() == null || ticketingSamplingRespVO.getPageSize() == null || ticketingSamplingRespVO.getTotalPages() == null || ticketingSamplingRespVO.getTotalRows() == null || ticketingSamplingRespVO.getDataMapList().isEmpty()) {
XxlJobHelper.log("{},请求抽数成功,但返回参数值:{}", date, ticketingSamplingRespVO);
XxlJobHelper.log("{},请求抽数成功,但返回参数存在问题{}", date, ticketingSamplingRespVO);
return null;
}
return ticketingSamplingRespVO;
}
/**
* 校验数据
* 请求失败消息
*
* @param date
* @return java.lang.Boolean
* @param
* @return void
*/
public Boolean dataValidation(String date) {
return checkTicketService.dataValidation(date) && saleDataService.dataValidation(date);
private void requestFailedMessage(TicketingSamplingRespVO ticketingSamplingRespVO) {
// 如果访问失败则让调度中心重新调用
if (!ticketingSamplingRespVO.isPassflag()) {
// 让调度中心重新调用
XxlJobHelper.log("抽数服务:检票数据抽数失败,返回消息:" + ticketingSamplingRespVO.getRtnMsg());
XxlJobHelper.handleFail("销售数据抽数失败,请重试");
}
}
}

View File

@ -56,11 +56,4 @@ public interface CheckTicketService {
List<String> deleteTwoYearLastMonthAgo();
List<String> findDistinctDate();
/**
* mongo和mysql数据数量验证
* @param date
* @return java.lang.Boolean
*/
Boolean dataValidation(String date);
}

View File

@ -149,9 +149,4 @@ public class CheckTicketServiceImpl implements CheckTicketService {
return checkTicketTodayRepository.findDistinctDate();
}
@Override
public Boolean dataValidation(String date) {
return checkTicketMapper.selectCount(CheckTicketDO::getCheckticketdate, date) == checkTicketTodayRepository.countByCustomDate(date);
}
}

View File

@ -55,11 +55,4 @@ public interface SaleDataService {
* @return java.util.List<java.lang.String>
*/
List<String> deleteTwoYearLastMonthAgo();
/**
* mongo和mysql数据数量验证
* @param date
* @return boolean
*/
Boolean dataValidation(String date);
}

View File

@ -136,9 +136,4 @@ public class SaleDataServiceImpl implements SaleDataService {
mongoTemplate.remove(query, "sale_data_today");
return filteredDate;
}
@Override
public Boolean dataValidation(String date) {
return Objects.equals(saleDataMapper.selectCount(SaleDataDO::getSddate, date), saleDataTodayRepository.countByCustomDate(date));
}
}