root/ludu-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request 'xinwei' (#75) from xinwei into master

Browse Source 
Reviewed-on: #75
This commit is contained in:
root 2024-09-13 10:46:57 +08:00
commit 84a34052b3
12 changed files with 381 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
.drone-images.yml Normal file
View File

@ -0,0 +1,145 @@
kind: pipeline # 定义对象类型，还有secret和signature两种类型
type: docker # 定义流水线类型，还有kubernetes、exec、ssh等类型
name: ludu-drone-images # 定义流水线名称
steps: # 定义流水线执行步骤，这些步骤将顺序执行
- name: package # 流水线名称
image: maven:3-jdk-8 # 定义创建容器的Docker镜像
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
- name: maven-cache
path: /root/.m2 # 将maven下载依赖的目录挂载出来，防止重复下载
- name: maven-build
path: /app/build # 将应用打包好的Jar和执行脚本挂载出来
commands: # 定义在Docker容器中执行的shell命令
- mvn package -Dmaven.test.skip=true # 应用打包命令
- |
if [ "${service}" = "gateway" ] || [ "${service}" = "all" ]; then
cp ./yudao-gateway/target/yudao-gateway.jar /app/build/yudao-gateway/target
cp ./yudao-gateway/Dockerfile /app/build/yudao-gateway
cp ./yudao-gateway/run.sh /app/build/yudao-gateway
fi
if [ "${service}" = "system" ] || [ "${service}" = "all" ]; then
cp ./yudao-module-system/yudao-module-system-biz/target/yudao-module-system-biz.jar /app/build/yudao-module-system/target
cp ./yudao-module-system/yudao-module-system-biz/Dockerfile /app/build/yudao-module-system
cp ./yudao-module-system/yudao-module-system-biz/run.sh /app/build/yudao-module-system
fi
if [ "${service}" = "infra" ] || [ "${service}" = "all" ]; then
cp ./yudao-module-infra/yudao-module-infra-biz/target/yudao-module-infra-biz.jar /app/build/yudao-module-infra/target
cp ./yudao-module-infra/yudao-module-infra-biz/Dockerfile /app/build/yudao-module-infra
cp ./yudao-module-infra/yudao-module-infra-biz/run.sh /app/build/yudao-module-infra
fi
if [ "${service}" = "job" ] || [ "${service}" = "all" ]; then
cp ./ludu-job-admin/ludu-job-admin-biz/target/ludu-job-admin-biz.jar /app/build/ludu-job-admin/target
cp ./ludu-job-admin/ludu-job-admin-biz/Dockerfile /app/build/ludu-job-admin
cp ./ludu-job-admin/ludu-job-admin-biz/run.sh /app/build/ludu-job-admin
fi
if [ "${service}" = "ticketing" ] || [ "${service}" = "all" ]; then
cp ./ludu-module-ticketing/ludu-module-ticketing-biz/target/ludu-module-ticketing-biz.jar /app/build/ludu-module-ticketing/target
cp ./ludu-module-ticketing/ludu-module-ticketing-biz/Dockerfile /app/build/ludu-module-ticketing
cp ./ludu-module-ticketing/ludu-module-ticketing-biz/run.sh /app/build/ludu-module-ticketing
fi
if [ "${service}" = "parking" ] || [ "${service}" = "all" ]; then
cp ./ludu-module-parking/ludu-module-parking-biz/target/ludu-module-parking-biz.jar /app/build/ludu-module-parking/target
cp ./ludu-module-parking/ludu-module-parking-biz/Dockerfile /app/build/ludu-module-parking
cp ./ludu-module-parking/ludu-module-parking-biz/run.sh /app/build/ludu-module-parking
fi
if [ "${service}" = "datacenter" ] || [ "${service}" = "all" ]; then
cp ./ludu-module-datacenter/ludu-module-datacenter-biz/target/ludu-module-datacenter-biz.jar /app/build/ludu-module-datacenter/target
cp ./ludu-module-datacenter/ludu-module-datacenter-biz/Dockerfile /app/build/ludu-module-datacenter
cp ./ludu-module-datacenter/ludu-module-datacenter-biz/run.sh /app/build/ludu-module-datacenter
fi
- name: push-images
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 101.43.112.107 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 30m # 远程执行命令超时时间
script:
- |
deploy_service() {
local image_name=$1
local container_name=$2
cd /ludu/build/${image_name}
docker stop ${container_name}
docker rm ${container_name}
docker rmi -f ${image_name}:1.0.0
docker buildx build -f Dockerfile -t ${image_name}:1.0.0 .
docker tag ${image_name}:1.0.0 120.46.37.243:8080/lundu/${image_name}:1.0.0
docker pull 120.46.37.243:8080/lundu/${image_name}:1.0.0
}
if [ "${service}" = "gateway" ] || [ "${service}" = "all" ]; then
deploy_service "yudao-gateway" "yudao-gateway"
fi
if [ "${service}" = "system" ] || [ "${service}" = "all" ]; then
deploy_service "yudao-module-system" "yudao-system"
fi
if [ "${service}" = "infra" ] || [ "${service}" = "all" ]; then
deploy_service "yudao-module-infra" "yudao-infra"
fi
if [ "${service}" = "job" ] || [ "${service}" = "all" ]; then
deploy_service "ludu-job-admin" "ludu-job-admin"
fi
if [ "${service}" = "ticketing" ] || [ "${service}" = "all" ]; then
deploy_service "ludu-module-ticketing" "ludu-ticketing"
fi
if [ "${service}" = "parking" ] || [ "${service}" = "all" ]; then
deploy_service "ludu-module-parking" "ludu-parking"
fi
if [ "${service}" = "datacenter" ] || [ "${service}" = "all" ]; then
deploy_service "ludu-module-datacenter" "ludu-datacenter"
fi
- name: pull-image-and-run
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /ludu
- ./start_service.sh ${service}
volumes: # 定义流水线挂载目录，用于共享数据
- name: maven-build
host:
path: /ludu/build # 从宿主机中挂载的目录
- name: maven-cache
host:
path: /ludu/cache # 从宿主机中挂载的目录

186
script/docker/docker-compose.yml
View File

@ -1,68 +1,82 @@
version: '3'
version: '3.8'
services:
yudao-gateway:
image: yudao-gateway
image: 120.46.37.243:8080/lundu/yudao-gateway:1.0.0
container_name: yudao-gateway
networks:
- ludu_network
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- JAVA_TOOL_OPTIONS=-javaagent:/data/skywalking/skywalking-agent/skywalking-agent.jar # 配置skywalking
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
- SW_AGENT_COLLECTOR_BACKEND_SERVICES=[YOUR_SKYWALKING_ADDR] # 请替换 your.skywalking.addr 为你的 skywalking 地址
- SPRING_PROFILES_ACTIVE=test # 指定程序运行环境
- SPRING_CLOUD_NACOS_CONFIG_SERVER_ADDR=[YOUR_NACOS_ADDR] # 配置中心地址
- SPRING_CLOUD_NACOS_CONFIG_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
- SPRING_CLOUD_NACOS_SERVER_ADDR=[YOUR_NACOS_ADDR] # 注册中心地址
- SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
volumes:
- /docker/yudao-cloud/logs:/root/logs/
- /data/skywalking/skywalking-agent:/data/skywalking/skywalking-agent
restart: always
network_mode: host # 以主机网络环境运行
healthcheck:
test: [ "CMD","curl","-f","http://localhost:48080" ]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
ports:
- "48083:48080"
yudao-system:
image: yudao-module-system-biz
image: 120.46.37.243:8080/lundu/yudao-module-system:1.0.0
container_name: yudao-system
networks:
- ludu_network
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- JAVA_TOOL_OPTIONS=-javaagent:/data/skywalking/skywalking-agent/skywalking-agent.jar # 配置skywalking
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
- SW_AGENT_COLLECTOR_BACKEND_SERVICES=[YOUR_SKYWALKING_ADDR] # 请替换 your.skywalking.addr 为你的 skywalking 地址
- SPRING_PROFILES_ACTIVE=test # 指定程序运行环境
- SPRING_CLOUD_NACOS_CONFIG_SERVER_ADDR=[YOUR_NACOS_ADDR] # 配置中心地址
- SPRING_CLOUD_NACOS_CONFIG_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
- SPRING_CLOUD_NACOS_SERVER_ADDR=[YOUR_NACOS_ADDR] # 注册中心地址
- SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
volumes:
- /docker/yudao-cloud/logs:/root/logs/
- /data/skywalking/skywalking-agent:/data/skywalking/skywalking-agent
healthcheck:
test: [ "CMD","curl","-f","http://localhost:48081" ]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
start_period: 120s
restart: always
network_mode: host
yudao-infra:
image: yudao-module-infra-biz
container_name: yudao-infra
depends_on:
yudao-gateway:
condition: service_healthy
ludu-job-admin:
image: 120.46.37.243:8080/lundu/ludu-job-admin:1.0.0
container_name: ludu-job-admin
networks:
- ludu_network
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
volumes:
- /docker/yudao-cloud/logs:/root/logs/
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:9090 || exit 0"]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
restart: always
ports:
- "48084:9090"
depends_on:
yudao-gateway:
condition: service_healthy
yudao-infra:
image: 120.46.37.243:8080/lundu/yudao-module-infra:1.0.0
container_name: yudao-infra
networks:
- ludu_network
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- JAVA_TOOL_OPTIONS=-javaagent:/data/skywalking/skywalking-agent/skywalking-agent.jar # 配置skywalking
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
- SW_AGENT_COLLECTOR_BACKEND_SERVICES=[YOUR_SKYWALKING_ADDR] # 请替换 your.skywalking.addr 为你的 skywalking 地址
- SPRING_PROFILES_ACTIVE=test # 指定程序运行环境
- SPRING_CLOUD_NACOS_CONFIG_SERVER_ADDR=[YOUR_NACOS_ADDR] # 配置中心地址
- SPRING_CLOUD_NACOS_CONFIG_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
- SPRING_CLOUD_NACOS_SERVER_ADDR=[YOUR_NACOS_ADDR] # 注册中心地址
- SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
volumes:
- /docker/yudao-cloud/logs:/root/logs/
- /data/skywalking/skywalking-agent:/data/skywalking/skywalking-agent
restart: always
network_mode: host
healthcheck:
test: [ "CMD","curl","-f","http://localhost:48082" ]
interval: 30s
@ -72,91 +86,75 @@ services:
depends_on:
yudao-system:
condition: service_healthy
yudao-report:
image: yudao-module-report-biz
container_name: yudao-report
yudao-gateway:
condition: service_healthy
ludu-datacenter:
image: 120.46.37.243:8080/lundu/ludu-module-datacenter:1.0.0
container_name: ludu-datacenter
networks:
- ludu_network
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- JAVA_TOOL_OPTIONS=-javaagent:/data/skywalking/skywalking-agent/skywalking-agent.jar # 配置skywalking
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
- SW_AGENT_COLLECTOR_BACKEND_SERVICES=[YOUR_SKYWALKING_ADDR] # 请替换 your.skywalking.addr 为你的 skywalking 地址
- SPRING_PROFILES_ACTIVE=test # 指定程序运行环境
- SPRING_CLOUD_NACOS_CONFIG_SERVER_ADDR=[YOUR_NACOS_ADDR] # 配置中心地址
- SPRING_CLOUD_NACOS_CONFIG_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
- SPRING_CLOUD_NACOS_SERVER_ADDR=[YOUR_NACOS_ADDR] # 注册中心地址
- SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
volumes:
- /docker/yudao-cloud/logs:/root/logs/
- /data/skywalking/skywalking-agent:/data/skywalking/skywalking-agent
restart: always
network_mode: host
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:48092 || exit 0"]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
depends_on:
yudao-infra:
yudao-gateway:
condition: service_healthy
yudao-bpm:
image: yudao-module-bpm-biz
container_name: yudao-bpm
ludu-parking:
image: 120.46.37.243:8080/lundu/ludu-module-parking:1.0.0
container_name: ludu-parking
networks:
- ludu_network
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- JAVA_TOOL_OPTIONS=-javaagent:/data/skywalking/skywalking-agent/skywalking-agent.jar # 配置skywalking
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
- SW_AGENT_COLLECTOR_BACKEND_SERVICES=[YOUR_SKYWALKING_ADDR] # 请替换 your.skywalking.addr 为你的 skywalking 地址
- SPRING_PROFILES_ACTIVE=test # 指定程序运行环境
- SPRING_CLOUD_NACOS_CONFIG_SERVER_ADDR=[YOUR_NACOS_ADDR] # 配置中心地址
- SPRING_CLOUD_NACOS_CONFIG_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
- SPRING_CLOUD_NACOS_SERVER_ADDR=[YOUR_NACOS_ADDR] # 注册中心地址
- SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
volumes:
- /docker/yudao-cloud/logs:/root/logs/
- /data/skywalking/skywalking-agent:/data/skywalking/skywalking-agent
restart: always
network_mode: host
healthcheck:
test: [ "CMD","curl","-f","http://localhost:48090" ]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
depends_on:
yudao-infra:
yudao-system:
condition: service_healthy
yudao-pay:
image: yudao-module-pay-biz
container_name: yudao-pay
yudao-gateway:
condition: service_healthy
ludu-ticketing:
image: 120.46.37.243:8080/lundu/ludu-module-ticketing:1.0.0
container_name: ludu-ticketing
networks:
- ludu_network
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- JAVA_TOOL_OPTIONS=-javaagent:/data/skywalking/skywalking-agent/skywalking-agent.jar # 配置skywalking
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
- SW_AGENT_COLLECTOR_BACKEND_SERVICES=[YOUR_SKYWALKING_ADDR] # 请替换 your.skywalking.addr 为你的 skywalking 地址
- SPRING_PROFILES_ACTIVE=test # 指定程序运行环境
- SPRING_CLOUD_NACOS_CONFIG_SERVER_ADDR=[YOUR_NACOS_ADDR] # 配置中心地址
- SPRING_CLOUD_NACOS_CONFIG_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
- SPRING_CLOUD_NACOS_SERVER_ADDR=[YOUR_NACOS_ADDR] # 注册中心地址
- SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
volumes:
- /docker/yudao-cloud/logs:/root/logs/
- /data/skywalking/skywalking-agent:/data/skywalking/skywalking-agent
restart: always
network_mode: host
healthcheck:
test: [ "CMD","curl","-f","http://localhost:48088" ]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
depends_on:
yudao-infra:
yudao-system:
condition: service_healthy
yudao-mp:
image: yudao-module-mp-biz
container_name: yudao-mp
environment:
- TZ=Asia/Shanghai # 配置程序默认时区为上海（中国标准时间）
- JAVA_TOOL_OPTIONS=-javaagent:/data/skywalking/skywalking-agent/skywalking-agent.jar # 配置skywalking
- SW_AGENT_NAME=yudao-gateway
- SW_AGENT_TRACE_IGNORE_PATH=Redisson/PING,/actuator/**,/admin/**
- SW_AGENT_COLLECTOR_BACKEND_SERVICES=[YOUR_SKYWALKING_ADDR] # 请替换 your.skywalking.addr 为你的 skywalking 地址
- SPRING_PROFILES_ACTIVE=test # 指定程序运行环境
- SPRING_CLOUD_NACOS_CONFIG_SERVER_ADDR=[YOUR_NACOS_ADDR] # 配置中心地址
- SPRING_CLOUD_NACOS_CONFIG_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
- SPRING_CLOUD_NACOS_SERVER_ADDR=[YOUR_NACOS_ADDR] # 注册中心地址
- SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE=[YOUR_NAMESPACE] # 命名空间
volumes:
- /docker/yudao-cloud/logs:/root/logs/
- /data/skywalking/skywalking-agent:/data/skywalking/skywalking-agent
restart: always
network_mode: host
depends_on:
yudao-infra:
yudao-gateway:
condition: service_healthy
networks:
ludu_network:
driver: bridge

23
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/ticketing/TicketingController.java
View File

@ -12,7 +12,6 @@ import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
@ -48,19 +47,30 @@ public class TicketingController {
endCalendar.setTime(TickingDateUtils.parse(endDate));
return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, endCalendar, method));
}
@GetMapping("/test1")
public CommonResult<List<String>> testAll(){
return CommonResult.success(checkTicketService.deleteTwoYearLastMonthAgo());
@GetMapping("/2")
public CommonResult<String> testTicketing2(@RequestParam("method") String method) throws Exception {
Calendar startCalendar = Calendar.getInstance();
startCalendar.add(Calendar.DAY_OF_MONTH, 1);
startCalendar.add(Calendar.YEAR, -1);
return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, startCalendar, method));
}
@GetMapping("/intoMysql")
public CommonResult<String> testTicketingMysql(@RequestParam("startDate") String startDate, @RequestParam("endDate") String endDate, @RequestParam("method") String method) throws Exception {
return CommonResult.success("");
}
private void methodUtils(Map<String, Object> map, String name, Object list) {
map.put(name, list);
}
private String methodDataUtils() {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
LocalDateTime now = LocalDateTime.now();
return now.format(formatter);
}
@GetMapping("/test")
public void test() {
Map<String, Object> map = new HashMap<>();
@ -75,8 +85,5 @@ public class TicketingController {
public CommonResult<List<String>> testAlla() {
return CommonResult.success(saleDataService.deleteTwoYearLastMonthAgo());
}
@GetMapping("/testSample")
public void testSample(){
ticketDataMigration.deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow();
}
}

7
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/checkticket/CheckTicketMapper.java
View File

@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.infra.controller.checkticket.vo.CheckTicketPageRe
import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketDO;
import com.baomidou.dynamic.datasource.annotation.DS;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
/**
* 检票 Mapper
@ -32,4 +33,10 @@ public interface CheckTicketMapper extends BaseMapperX<CheckTicketDO> {
.orderByDesc(CheckTicketDO::getId));
}
/**
* 根据检票日期物理删除数据
* @param checkticketdate
* @return java.lang.Integer
*/
Integer physicalDeleteByCheckticketdate(@Param("checkticketdate") String checkticketdate);
}

12
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/saledata/SaleDataMapper.java
View File

@ -6,10 +6,7 @@ import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.infra.controller.saledata.vo.SaleDataPageReqVO;
import cn.iocoder.yudao.module.infra.dal.dataobject.saledata.SaleDataDO;
import com.baomidou.dynamic.datasource.annotation.DS;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Result;
import org.apache.ibatis.annotations.Results;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.*;
import java.util.List;
import java.util.Map;
@ -43,4 +40,11 @@ public interface SaleDataMapper extends BaseMapperX<SaleDataDO> {
.likeIfPresent(SaleDataDO::getSalepropetyvaluename, reqVO.getSalepropetyvaluename())
.orderByDesc(SaleDataDO::getId));
}
/**
* 根据销售日期物理删除
* @param sddate
* @return java.lang.Integer
*/
Integer physicalDeleteBySddate(@Param("sddate") String sddate);
}

323
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/job/ticketing/TicketDataMigration.java
View File

@ -59,274 +59,119 @@ public class TicketDataMigration {
* 存入售票数据的定时任务mysql
*/
@XxlJob("getSaleData")
public void getSaleDataHandler() throws Exception {
public String getSaleDataHandler() throws Exception {
int failedCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务：mysql销售数据开始抽数...");
StringJoiner result = new StringJoiner("\n");
XxlJobHelper.log("抽数服务：mysql售票数据开始抽数...");
result.add("抽数服务：mysql售票数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
XxlJobHelper.log("日期：{}暂无数据！", TickingDateUtils.getPreviousDayFormat());
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "暂无数据！");
return result.toString();
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
if (totalRows <= PAGE_SIZE) {
failedCount += insertSaleData(mapList, totalRows, pageNumber);
} else {
// 总条数大于每页条数 执行完第一次插入后根据返回的页数继续进行请求和插入
// 插入第一页的数据
failedCount += insertSaleData(mapList, PAGE_SIZE, pageNumber);
pageNumber++;
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理，得到最后一页的实际条数
if (pageNumber != totalPages) {
failedCount += insertSaleData(mapList, PAGE_SIZE, pageNumber);
} else {
failedCount += insertSaleData(mapList, totalRows % PAGE_SIZE, pageNumber);
if (!saleDataService.createSaleData(BeanUtils.toBean(mapList, SaleDataDO.class))) {
failedCount += ticketingSamplingRespVO.getPageSize();
}
}
if (failedCount != 0) {
XxlJobHelper.handleFail("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
result.add("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
return result.toString();
}
if (!saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致！！！");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getSaleDataByMongoDBHandler();
XxlJobHelper.log("抽数完，比对结果：{}", saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
XxlJobHelper.log("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
result.add("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
Calendar yesterday = Calendar.getInstance();
yesterday.add(Calendar.DAY_OF_MONTH, -1);
getTicketDataByTimeZonesToMongo(yesterday, yesterday, "saleData");
if (!saleDataService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.handleFail("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql售票数据不一致！！！");
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql售票数据不一致！！！");
}
XxlJobHelper.log("抽数服务：销售数据抽数正常结束！总共：{}条数据，{}条数据插入失败！", totalRows, failedCount);
}
XxlJobHelper.log("抽数服务：售票数据抽数正常结束！总共：{}条数据，{}条数据插入失败！", totalRows, failedCount);
result.add("抽数服务：售票数据抽数正常结束！总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！");
return result.toString();
}
/**
* 存入检票数据的定时任务mysql
*/
@XxlJob("getCheckTicket")
public void getCheckTicketHandler() throws Exception {
public String getCheckTicketHandler() throws Exception {
int failedCount = 0;
int pageNumber = 1;
StringJoiner result = new StringJoiner("\n");
result.add("抽数服务：mysql检票数据开始抽数...");
XxlJobHelper.log("抽数服务：mysql检票数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
XxlJobHelper.log("日期：{}暂无数据！", TickingDateUtils.getPreviousDayFormat());
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "暂无数据！");
return result.toString();
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList = ticketingSamplingRespVO.getDataMapList();
// 总条数小于等于每页条数 则直接按照总条数进行插入即可
if (totalRows <= PAGE_SIZE) {
failedCount += insertCheckTicket(mapList, totalRows, pageNumber);
} else {
// 总条数大于每页条数 执行完第一次插入后根据返回的页数继续进行请求和插入
// 插入第一页的数据
failedCount += insertCheckTicket(mapList, PAGE_SIZE, pageNumber);
pageNumber++;
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, TickingDateUtils.getPreviousDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
// 对最后一页进行处理，得到最后一页的实际条数
if (pageNumber != totalPages) {
failedCount += insertCheckTicket(mapList, PAGE_SIZE, pageNumber);
} else {
failedCount += insertCheckTicket(mapList, totalRows % PAGE_SIZE, pageNumber);
if (!checkTicketService.createCheckTicket(BeanUtils.toBean(mapList, CheckTicketDO.class))) {
failedCount += ticketingSamplingRespVO.getPageSize();
}
}
if (failedCount != 0) {
XxlJobHelper.handleFail("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
result.add("总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！即将重试......");
return result.toString();
}
if (!checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.log("mysql和mongo数据验证不一致！！！");
XxlJobHelper.log("重新抽取({})mongo数据", TickingDateUtils.getPreviousDayFormat());
getCheckTicketByMongoDBHandler();
XxlJobHelper.log("抽数完，比对结果：{}", checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat()) ? "相等" : "不想等");
XxlJobHelper.log("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
result.add("mysql和mongo数据验证不一致！！！重新进行mongo抽数中......");
Calendar yesterday = Calendar.getInstance();
yesterday.add(Calendar.DAY_OF_MONTH, -1);
getTicketDataByTimeZonesToMongo(yesterday, yesterday, "checkTicket");
if (!checkTicketService.dataValidation(TickingDateUtils.getPreviousDayFormat())) {
XxlJobHelper.handleFail("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql检票数据不一致！！！");
result.add("日期：" + TickingDateUtils.getPreviousDayFormat() + "中mongo和mysql检票数据不一致！！！");
}
}
XxlJobHelper.log("抽数服务：检票数据抽数正常结束！总共：{}条数据，{}条数据插入失败！", totalRows, failedCount);
result.add("抽数服务：检票数据抽数正常结束！总共：" + totalRows + "条数据，" + failedCount + "条数据插入失败！");
return result.toString();
}
/**
* 存入销售数据的定时任务mongodb
*/
@XxlJob("getSaleDataByMongoDB")
public void getSaleDataByMongoDBHandler() throws Exception {
int successCount = 0;
int duplicatesCount = 0;
int pageNumber = 1;
XxlJobHelper.log("抽数服务：mongodb销售数据开始抽数...");
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, TickingDateUtils.getNowDayFormat(), pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = saleDataService.insertTodayByMongoDB(BeanUtils.toBean(mapList, SaleDataToday.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
}
XxlJobHelper.log("抽数服务：检票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", totalRows, duplicatesCount, successCount);
public String getSaleDataByMongoDBHandler() throws Exception {
Calendar now = Calendar.getInstance();
return getTicketDataByTimeZonesToMongo(now, now, "saleData");
}
/**
* 存入检票数据的定时任务mongodb
*/
@XxlJob("getCheckTicketByMongoDB")
public void getCheckTicketByMongoDBHandler() throws Exception {
XxlJobHelper.log("抽数服务：mongodb检票数据开始抽数...");
int successCount = 0;
int duplicatesCount = 0;
int pageNumber = 1;
String date = TickingDateUtils.getNowDayFormat();
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, date, pageNumber);
if (ticketingSamplingRespVO == null) {
XxlJobHelper.log("暂无数据！");
return;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, date, pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = checkTicketService.insertTodayByMongoDB(BeanUtils.toBean(mapList, CheckTicketToday.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
public String getCheckTicketByMongoDBHandler() throws Exception {
Calendar now = Calendar.getInstance();
return getTicketDataByTimeZonesToMongo(now, now, "checkTicket");
}
XxlJobHelper.log("抽数服务：检票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", totalRows, duplicatesCount, successCount);
}
@XxlJob("getCheckTicketByMongoDBLastMonthToNow")
public void getCheckTicketByMongoDBLastMonthToNow() throws Exception {
XxlJobHelper.log("抽数服务：mongodb检票数据开始抽数...");
int successCount;
int duplicatesCount;
int pageNumber;
// 获取当前日期
Calendar startDate = Calendar.getInstance();
// 设置开始日期为上一个月的第一天
startDate.set(Calendar.DAY_OF_MONTH, 1); // 先将日期设置为当前月的第一天
startDate.add(Calendar.MONTH, -1); // 然后减去一个月
// 设置结束日期为今天
Calendar endDate = Calendar.getInstance();
// 循环从开始日期到结束日期
while (!startDate.after(endDate)) {
successCount = 0;
duplicatesCount = 0;
String dateString = TickingDateUtils.format(startDate.getTime());
pageNumber = 1;
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, dateString, pageNumber);
if (ticketingSamplingRespVO == null) {
startDate.add(Calendar.DAY_OF_MONTH, 1);
continue;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(CHECK_TICKET_SERVICE, CHECK_TICKET_METHOD, dateString, pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = checkTicketService.insertByMongoDB(BeanUtils.toBean(mapList, CheckTicket.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
}
int currentMonth = startDate.get(Calendar.MONTH) + 1; // 加1以得到1-12的月份
int currentDay = startDate.get(Calendar.DAY_OF_MONTH);
XxlJobHelper.log("抽数服务：{}月{}日：检票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", currentMonth, currentDay, totalRows, duplicatesCount, successCount);
// 日期递增一天
startDate.add(Calendar.DAY_OF_MONTH, 1);
}
XxlJobHelper.log("抽数结束!!!");
}
@XxlJob("getSaleDataByMongoDBLastMonthToNow")
public void getSaleDataByMongoDBLastMonthToNow() throws Exception {
XxlJobHelper.log("抽数服务：mongodb售票数据开始抽数...");
int successCount;
int duplicatesCount;
int pageNumber;
// 获取当前日期
Calendar startDate = Calendar.getInstance();
// 设置开始日期为上一个月的第一天
startDate.set(Calendar.DAY_OF_MONTH, 1); // 先将日期设置为当前月的第一天
startDate.add(Calendar.MONTH, -1); // 然后减去一个月
// 设置结束日期为今天
Calendar endDate = Calendar.getInstance();
// 循环从开始日期到结束日期
while (!startDate.after(endDate)) {
successCount = 0;
duplicatesCount = 0;
String dateString = TickingDateUtils.format(startDate.getTime());
pageNumber = 1;
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, dateString, pageNumber);
if (ticketingSamplingRespVO == null) {
startDate.add(Calendar.DAY_OF_MONTH, 1);
continue;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
List<Map<String, Object>> mapList;
// 获得总页数
int totalPages = ticketingSamplingRespVO.getTotalPages();
// 处理剩余页的数据
for (; pageNumber <= totalPages; pageNumber++) {
ticketingSamplingRespVO = this.getUrlResponseData(SALE_DATA_SERVICE, SALE_DATA_METHOD, dateString, pageNumber);
mapList = ticketingSamplingRespVO.getDataMapList();
List<Integer> resultNum = saleDataService.insertByMongoDB(BeanUtils.toBean(mapList, SaleData.class));
if (resultNum == null) {
XxlJobHelper.log("数据批量插入异常！", pageNumber);
} else {
successCount += resultNum.get(0);
duplicatesCount += resultNum.get(1);
}
}
int currentMonth = startDate.get(Calendar.MONTH) + 1; // 加1以得到1-12的月份
int currentDay = startDate.get(Calendar.DAY_OF_MONTH);
XxlJobHelper.log("抽数服务：{}月{}日：售票数据抽数正常结束！总共：{}条数据，{}条数据重复，{}条数据插入成功！", currentMonth, currentDay, totalRows, duplicatesCount, successCount);
// 日期递增一天
startDate.add(Calendar.DAY_OF_MONTH, 1);
}
XxlJobHelper.log("抽数结束!!!");
}
/**
* 删除（今年和去年）往前推31天外的数据和抽取去年明天的数据
@ -396,6 +241,10 @@ public class TicketDataMigration {
TicketingSamplingRespVO ticketingSamplingRespVO = this.getUrlResponseData(respService, respMethod, dateString, pageNumber);
if (ticketingSamplingRespVO == null) {
startDate.add(Calendar.DAY_OF_MONTH, 1);
result.add("日期：" + TickingDateUtils.format(startDate.getTime()) + "返回无值");
if (startDate.equals(endDate)) { // 如果日期相等，退出循环
break;
}
continue;
}
int totalRows = ticketingSamplingRespVO.getTotalRows();
@ -437,48 +286,6 @@ public class TicketDataMigration {
return result.toString();
}
/**
* 插入销售数据的方法（mysql）
*
* @param mapList 数据集合数组
* @param listSize 实际个数
* @param pageNumber 每页显示条数
* @return int 失败条数
*/
private int insertSaleData(List<Map<String, Object>> mapList, int listSize, int pageNumber) {
int failedCount = 0;
for (int i = 0; i < listSize; i++) {
Map<String, Object> currentMap = mapList.get(i);
SaleDataDO saleDataDO = BeanUtils.toBean(currentMap, SaleDataDO.class);
if (saleDataService.createSaleData(saleDataDO) == 0) {
failedCount++;
XxlJobHelper.log("第{}条数据插入失败，数据值为：{}", ((pageNumber - 1) * PAGE_SIZE + i + 1), currentMap);
}
}
return failedCount;
}
/**
* 插入检票数据的方法（mysql）
*
* @param mapList 数据集合数组
* @param listSize 实际个数
* @param pageNumber 每页显示条数
* @return int 失败条数
*/
private int insertCheckTicket(List<Map<String, Object>> mapList, int listSize, int pageNumber) {
int failedCount = 0;
for (int i = 0; i < listSize; i++) {
Map<String, Object> currentMap = mapList.get(i);
CheckTicketDO checkTicketDO = BeanUtils.toBean(currentMap, CheckTicketDO.class);
if (checkTicketService.createCheckTicket(checkTicketDO) == 0) {
failedCount++;
XxlJobHelper.log("第{}条数据插入失败，数据值为：{}", ((pageNumber - 1) * PAGE_SIZE + i + 1), currentMap);
}
}
return failedCount;
}
/**
* 请求目标url的响应数据
*
@ -519,14 +326,4 @@ public class TicketDataMigration {
return ticketingSamplingRespVO;
}
/**
* 校验数据
*
* @param date
* @return java.lang.Boolean
*/
public Boolean dataValidation(String date) {
return checkTicketService.dataValidation(date) && saleDataService.dataValidation(date);
}
}

6
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketService.java
View File

@ -19,12 +19,10 @@ import java.util.Map;
public interface CheckTicketService {
/**
* 创建检票
*
* @param createReqVO 创建信息
* 批量插入检票数据(mysql)
* @return 编号
*/
Long createCheckTicket(@Valid CheckTicketDO createReqVO);
Boolean createCheckTicket(List<CheckTicketDO> checkTicketDOList);
/**
* 查询重复的dataId(mongodb)

7
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketServiceImpl.java
View File

@ -48,11 +48,10 @@ public class CheckTicketServiceImpl implements CheckTicketService {
private MongoTemplate mongoTemplate;
@Override
public Long createCheckTicket(CheckTicketDO checkTicketDO) {
// 插入
checkTicketMapper.insert(checkTicketDO);
public Boolean createCheckTicket(List<CheckTicketDO> checkTicketDOList) {
checkTicketMapper.physicalDeleteByCheckticketdate(checkTicketDOList.get(0).getCheckticketdate());
// 返回
return checkTicketDO.getId();
return checkTicketMapper.insertBatch(checkTicketDOList);
}
@Override

6
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataService.java
View File

@ -21,12 +21,12 @@ import java.util.Map;
public interface SaleDataService {
/**
* 创建售票
* 批量插入售票数据(mysql)
*
* @param saleDataDO 创建信息
* @param saleDataDOList
* @return 编号
*/
Long createSaleData(@Valid SaleDataDO saleDataDO);
Boolean createSaleData(List<SaleDataDO> saleDataDOList);
/**
* 查询重复的dataId(mongodb)

8
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataServiceImpl.java
View File

@ -42,11 +42,9 @@ public class SaleDataServiceImpl implements SaleDataService {
private MongoTemplate mongoTemplate;
@Override
public Long createSaleData(SaleDataDO saleDataDO) {
// 插入
saleDataMapper.insert(saleDataDO);
// 返回
return saleDataDO.getId();
public Boolean createSaleData(List<SaleDataDO> saleDataDOList) {
saleDataMapper.physicalDeleteBySddate(saleDataDOList.get(0).getSddate());
return saleDataMapper.insertBatch(saleDataDOList);
}

9
yudao-module-infra/yudao-module-infra-biz/src/main/resources/mapper/checkticket/CheckticketMapper.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.iocoder.yudao.module.infra.dal.mysql.checkticket.CheckTicketMapper">
<delete id="physicalDeleteByCheckticketdate">
DELETE
FROM checkticket
WHERE checkticketdate = #{checkticketdate}
</delete>
</mapper>

9
yudao-module-infra/yudao-module-infra-biz/src/main/resources/mapper/saledata/SaleDataMapper.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.iocoder.yudao.module.infra.dal.mysql.saledata.SaleDataMapper">
<delete id="physicalDeleteBySddate">
DELETE
FROM saledata
WHERE sddate = #{sddate}
</delete>
</mapper>