root/ludu-cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

xinwei #59

Merged
root merged 2 commits from xinwei into master 2024-08-29 16:38:04 +08:00
Conversation 0 Commits 2 Files Changed 11 +252 -53
11 changed files with 252 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.drone-base.yml
View File

@ -98,11 +98,11 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
- cd /ludu/maven/build/yudao-gateway/
- chmod +x ./run.sh # 更改为可执行脚本
- ./run.sh || echo "ludu-module-ticketing build failed" # 运行脚本打包应用镜像并运行
- ./run.sh || echo "yudao-gateway build failed" # 运行脚本打包应用镜像并运行
- cd /ludu/maven/build/yudao-module-system/
- chmod +x ./run.sh # 更改为可执行脚本
- ./run.sh || echo "ludu-module-datacenter build failed" # 运行脚本打包应用镜像并运行
- ./run.sh || echo "yudao-module-system build failed" # 运行脚本打包应用镜像并运行
- cd /ludu/maven/build/yudao-module-infra/
- chmod +x ./run.sh # 更改为可执行脚本

91
.drone-infra.yml Normal file
View File

@ -0,0 +1,91 @@
kind: pipeline # 定义对象类型，还有secret和signature两种类型
type: docker # 定义流水线类型，还有kubernetes、exec、ssh等类型
name: ludu-system-drone # 定义流水线名称
steps: # 定义流水线执行步骤，这些步骤将顺序执行
- name: package # 流水线名称
image: maven:3-jdk-8 # 定义创建容器的Docker镜像
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
- name: maven-cache
path: /root/.m2 # 将maven下载依赖的目录挂载出来，防止重复下载
- name: maven-build
path: /app/build # 将应用打包好的Jar和执行脚本挂载出来
commands: # 定义在Docker容器中执行的shell命令
- mvn package -Dmaven.test.skip=true # 应用打包命令
- mkdir -p /app/build/yudao-module-infra/target
- cp ./yudao-module-infra/yudao-module-infra-biz/target/yudao-module-infra-biz.jar /app/build/yudao-module-infra/target
- cp ./yudao-module-infra/yudao-module-infra-biz/Dockerfile /app/build/yudao-module-infra
- cp ./yudao-module-infra/yudao-module-infra-biz/run.sh /app/build/yudao-module-infra
- name: copy-to-object # 构建基础模块和复制票务停车场到轮渡服务器
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 101.43.112.107 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 15m # 远程执行命令超时时间
script:
- echo "start copy"
- scp -r /ludu/maven/build/yudao-module-infra root@121.36.203.133:/ludu/maven/build
- echo "infra done"
- name: object-build
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 121.36.203.133 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password3 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 20m # 远程执行命令超时时间
script:
- cd /ludu/maven/build/yudao-module-infra/
- chmod +x ./run.sh # 更改为可执行脚本
- ./run.sh || echo "yudao-module-infra build failed" # 运行脚本打包应用镜像并运行
volumes: # 定义流水线挂载目录，用于共享数据
- name: maven-build
host:
path: /ludu/maven/build # 从宿主机中挂载的目录
- name: maven-cache
host:
path: /ludu/maven/cache # 从宿主机中挂载的目录

60
.drone.yml
View File

@ -55,10 +55,10 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
- cp ./ludu-module-parking/ludu-module-parking-biz/Dockerfile /app/build/ludu-module-parking
- cp ./ludu-module-parking/ludu-module-parking-biz/run.sh /app/build/ludu-module-parking
- mkdir -p /app/build/ludu-module-datacenter/target
- cp ./ludu-module-datacenter/ludu-module-datacenter-biz/target/ludu-module-datacenter-biz.jar /app/build/ludu-module-datacenter/target
- cp ./ludu-module-datacenter/ludu-module-datacenter-biz/Dockerfile /app/build/ludu-module-datacenter
- cp ./ludu-module-datacenter/ludu-module-datacenter-biz/run.sh /app/build/ludu-module-datacenter
# - mkdir -p /app/build/ludu-module-datacenter/target
# - cp ./ludu-module-datacenter/ludu-module-datacenter-biz/target/ludu-module-datacenter-biz.jar /app/build/ludu-module-datacenter/target
# - cp ./ludu-module-datacenter/ludu-module-datacenter-biz/Dockerfile /app/build/ludu-module-datacenter
# - cp ./ludu-module-datacenter/ludu-module-datacenter-biz/run.sh /app/build/ludu-module-datacenter
- name: copy-to-object # 构建基础模块和复制票务停车场到轮渡服务器
@ -93,32 +93,32 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
- echo "infra done"
- scp -r /ludu/maven/build/ludu-job-admin root@121.36.203.133:/ludu/maven/build
- echo "job done"
- scp -r /ludu/maven/build/ludu-module-datacenter root@120.46.37.243:/ludu/maven/build
- echo "datacenter done"
# - scp -r /ludu/maven/build/ludu-module-datacenter root@120.46.37.243:/ludu/maven/build
# - echo "datacenter done"
- name: build-other-service
image: appleboy/drone-ssh # SSH工具镜像
settings:
host: 120.46.37.243 # 远程连接地址
username: root # 远程连接账号
password:
from_secret: ssh_password2 # 从Secret中读取SSH密码
port: 22 # 远程连接端口
command_timeout: 5m # 远程执行命令超时时间
script:
- cd /ludu/maven/build/ludu-module-datacenter/
- chmod +x ./run.sh # 更改为可执行脚本
- ./run.sh || echo "ludu-module-datacenter build failed" # 运行脚本打包应用镜像并运行
# - name: build-other-service
#
# image: appleboy/drone-ssh # SSH工具镜像
#
# settings:
#
# host: 120.46.37.243 # 远程连接地址
#
# username: root # 远程连接账号
#
# password:
#
# from_secret: ssh_password2 # 从Secret中读取SSH密码
#
# port: 22 # 远程连接端口
#
# command_timeout: 5m # 远程执行命令超时时间
#
# script:
#
# - cd /ludu/maven/build/ludu-module-datacenter/
# - chmod +x ./run.sh # 更改为可执行脚本
# - ./run.sh || echo "ludu-module-datacenter build failed" # 运行脚本打包应用镜像并运行
- name: object-build
image: appleboy/drone-ssh # SSH工具镜像
@ -190,7 +190,7 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
echo "Killing process $pid"
kill -9 $pid
fi
nohup java -Xms512m -Xmx512m -jar ludu-module-datacenter/ludu-module-datacenter-biz/target/ludu-module-datacenter-biz.jar --spring.profiles.active=dev > api.log &
nohup java -jar ludu-module-datacenter/ludu-module-datacenter-biz/target/ludu-module-datacenter-biz.jar --spring.profiles.active=dev > api.log &
volumes: # 定义流水线挂载目录，用于共享数据
- name: maven-build

17
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/ticketing/TicketingController.java
View File

@ -3,6 +3,8 @@ package cn.iocoder.yudao.module.infra.controller.admin.ticketing;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
import cn.iocoder.yudao.module.infra.job.ticketing.TicketDataMigration;
import cn.iocoder.yudao.module.infra.service.checkticket.CheckTicketService;
import cn.iocoder.yudao.module.infra.service.saledata.SaleDataService;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -10,6 +12,7 @@ import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
/**
* @Description 测试抽数接口
@ -22,6 +25,12 @@ public class TicketingController {
@Resource
TicketDataMigration ticketDataMigration;
@Resource
CheckTicketService checkTicketService;
@Resource
SaleDataService saleDataService;
@GetMapping
public CommonResult<String> testTicketing(@RequestParam("startDate") String startDate, @RequestParam("endDate")String endDate, @RequestParam("method") String method) throws Exception {
Calendar startCalendar = Calendar.getInstance();
@ -30,4 +39,12 @@ public class TicketingController {
endCalendar.setTime(TickingDateUtils.parse(endDate));
return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, endCalendar, method));
}
@GetMapping("/test")
public CommonResult<List<String>> testAll(){
return CommonResult.success(checkTicketService.deleteTwoYearLastMonthAgo());
}
@GetMapping("/testa")
public CommonResult<List<String>> testAlla(){
return CommonResult.success(saleDataService.deleteTwoYearLastMonthAgo());
}
}

17
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mongodb/checkticket/CheckTicketTodayRepository.java
View File

@ -3,6 +3,7 @@ package cn.iocoder.yudao.module.infra.dal.mongodb.checkticket;
import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket;
import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketToday;
import org.springframework.data.mongodb.repository.Aggregation;
import org.springframework.data.mongodb.repository.MongoRepository;
import org.springframework.data.mongodb.repository.Query;
import org.springframework.stereotype.Repository;
@ -27,4 +28,20 @@ public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketT
*/
@Query("{ 'checkticketdate': { $not: { $gte: ?0, $lt: ?1 } } }")
List<CheckTicketToday> findNotInTimeRange(String startDate, String endDate);
/**
* 查询所有去重的 checkticketdate 字段值
* @return 去重后的 checkticketdate 字段值列表
*/
@Aggregation(pipeline = {
"{$group: {_id: '$checkticketdate'}}",
"{$project: {_id: 0, checkticketdate: '$_id'}}"
})
List<String> findDistinctDate();
/**
* 删除 checkticketdate 字段值在给定列表中的所有记录
* @param dates 日期列表
*/
@Query("{ 'checkticketdate': { $in: ?0 } }")
void deleteByCheckticketdateIn(List<String> dates);
}

9
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mongodb/saledata/SaleDataTodayRepository.java
View File

@ -78,4 +78,13 @@ public interface SaleDataTodayRepository extends MongoRepository<SaleDataToday,S
*/
@Query("{ 'sddate': { $not: { $gte: ?0, $lt: ?1 } } }")
List<SaleDataToday> findNotInTimeRange(String startDate, String endDate);
/**
* 查询所有去重的 sddate 字段值
* @return 去重后的 sddate 字段值列表
*/
@Aggregation(pipeline = {
"{$group: {_id: '$sddate'}}",
"{$project: {_id: 0, sddate: '$_id'}}"
})
List<String> findDistinctDate();
}

28
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/job/ticketing/TicketDataMigration.java
View File

@ -309,35 +309,23 @@ public class TicketDataMigration {
@XxlJob("deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow")
public void deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow(){
XxlJobHelper.log("删除多余数据开始...");
// 检票数据今年多余删除
Calendar today = Calendar.getInstance();
Calendar thirtyDaysAgo = (Calendar) today.clone(); // 克隆当前日期
thirtyDaysAgo.add(Calendar.DAY_OF_MONTH, -31);
List<String> thisYearCheckTicketList = checkTicketService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(thirtyDaysAgo.getTime()), TickingDateUtils.format(today.getTime()));
XxlJobHelper.log("删除今年检票日期为{}", thisYearCheckTicketList.toString());
// 检票数据去年多余删除
Calendar lastYearToday = (Calendar) today.clone();
lastYearToday.add(Calendar.YEAR, -1);
Calendar lastYearThirtyDaysAgo = (Calendar) lastYearToday.clone();
lastYearThirtyDaysAgo.add(Calendar.DAY_OF_MONTH, -31);
List<String> lastYearCheckTicketList = checkTicketService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(lastYearThirtyDaysAgo.getTime()), TickingDateUtils.format(lastYearToday.getTime()));
XxlJobHelper.log("删除去年检票日期为{}", lastYearCheckTicketList.toString());
// 售票数据今年多余删除
List<String> thisYearSaleDataList = saleDataService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(thirtyDaysAgo.getTime()), TickingDateUtils.format(today.getTime()));
XxlJobHelper.log("删除今年售票日期为{}", thisYearSaleDataList.toString());
// 售票数据去年多余删除
List<String> lastYearSaleDataList = checkTicketService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(lastYearThirtyDaysAgo.getTime()), TickingDateUtils.format(lastYearToday.getTime()));
XxlJobHelper.log("删除去年检票日期为{}", lastYearSaleDataList.toString());
// 检票数据今年去年多余删除
XxlJobHelper.log("删除检票日期为{}", checkTicketService.deleteTwoYearLastMonthAgo().toString());
// 售票数据今年去年多余删除
XxlJobHelper.log("删除售票日期为{}", saleDataService.deleteTwoYearLastMonthAgo().toString());
// 抽取去年明天的检票和售票数据
Calendar lastYearTomorrow = (Calendar) lastYearToday.clone();
Calendar lastYearTomorrow = Calendar.getInstance();
lastYearTomorrow.add(Calendar.DAY_OF_MONTH, 1);
lastYearTomorrow.add(Calendar.YEAR, -1);
try {
this.getTicketDataByTimeZonesToMongo(lastYearTomorrow, lastYearTomorrow, "checkTicket");
XxlJobHelper.log("抽取去年明天的检票数据成功，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
} catch (Exception e) {
XxlJobHelper.log("抽取去年明天的检票数据失败，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
}
try {
this.getTicketDataByTimeZonesToMongo(lastYearTomorrow, lastYearTomorrow, "saleData");
XxlJobHelper.log("抽取去年明天的售票数据成功，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
} catch (Exception e) {
XxlJobHelper.log("抽取去年明天的售票数据失败，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
}

9
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketService.java
View File

@ -47,4 +47,13 @@ public interface CheckTicketService {
* @return java.util.List<java.lang.String> 删除的日期
*/
List<String> deleteNotInTimeZonesByMongoDB(String startDate,String endDate);
/**
* 删除两年上个月以前的数据(mongodb)
* @param
* @return java.util.List<java.lang.String>
*/
List<String> deleteTwoYearLastMonthAgo();
List<String> findDistinctDate();
}

33
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketServiceImpl.java
View File

@ -1,6 +1,7 @@
package cn.iocoder.yudao.module.infra.service.checkticket;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket;
@ -14,6 +15,7 @@ import com.baomidou.dynamic.datasource.annotation.Slave;
import com.mongodb.MongoException;
import org.springframework.data.domain.Example;
import org.springframework.data.domain.ExampleMatcher;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
@ -42,6 +44,8 @@ public class CheckTicketServiceImpl implements CheckTicketService {
private CheckTicketRepository checkTicketRepository;
@Resource
private CheckTicketTodayRepository checkTicketTodayRepository;
@Resource
private MongoTemplate mongoTemplate;
@Override
public Long createCheckTicket(CheckTicketDO checkTicketDO) {
@ -116,4 +120,33 @@ public class CheckTicketServiceImpl implements CheckTicketService {
return deletedDates;
}
@Override
public List<String> deleteTwoYearLastMonthAgo() {
List<String> totalDate = checkTicketTodayRepository.findDistinctDate();
List<String> exitDate = new ArrayList<>();
Calendar endDate = Calendar.getInstance();
Calendar endDateLastYear = (Calendar) endDate.clone();
endDateLastYear.add(Calendar.YEAR, -1);
for (int i = 0; i < 31; i++) {
exitDate.add(TickingDateUtils.format(endDate.getTime()));
exitDate.add(TickingDateUtils.format(endDateLastYear.getTime()));
endDate.add(Calendar.DAY_OF_MONTH, -1);
endDateLastYear.add(Calendar.DAY_OF_MONTH, -1);
}
List<String> filteredDate = totalDate.stream()
.filter(date -> !exitDate.contains(date))
.collect(Collectors.toList());
// TODO 使用这种方法删除会报错，暂时使用mongoTemplate进行删除
// checkTicketTodayRepository.deleteByCheckticketdateIn(filteredDate);
Query query = new Query();
query.addCriteria(Criteria.where("sddate").in(filteredDate));
mongoTemplate.remove(query, "sale_data_today");
return filteredDate;
}
@Override
public List<String> findDistinctDate() {
return checkTicketTodayRepository.findDistinctDate();
}
}

6
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataService.java
View File

@ -49,4 +49,10 @@ public interface SaleDataService {
* @return java.util.List<java.lang.String> 删除的日期
*/
List<String> deleteNotInTimeZonesByMongoDB(String startDate,String endDate);
/**
* 删除两年上个月以前的数据(mongodb)
* @param
* @return java.util.List<java.lang.String>
*/
List<String> deleteTwoYearLastMonthAgo();
}

31
yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataServiceImpl.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.infra.service.saledata;
import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket;
import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketToday;
import cn.iocoder.yudao.module.infra.dal.dataobject.saledata.SaleData;
@ -9,6 +10,9 @@ import cn.iocoder.yudao.module.infra.dal.mongodb.saledata.SaleDataRepository;
import cn.iocoder.yudao.module.infra.dal.mongodb.saledata.SaleDataTodayRepository;
import cn.iocoder.yudao.module.infra.dal.mysql.saledata.SaleDataMapper;
import com.mongodb.MongoException;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
@ -34,6 +38,9 @@ public class SaleDataServiceImpl implements SaleDataService {
@Resource
private SaleDataMapper saleDataMapper;
@Resource
private MongoTemplate mongoTemplate;
@Override
public Long createSaleData(SaleDataDO saleDataDO) {
// 插入
@ -106,5 +113,27 @@ public class SaleDataServiceImpl implements SaleDataService {
.collect(Collectors.toList());
return deletedDates;
}
@Override
public List<String> deleteTwoYearLastMonthAgo() {
List<String> totalDate = saleDataTodayRepository.findDistinctDate();
List<String> exitDate = new ArrayList<>();
Calendar endDate = Calendar.getInstance();
Calendar endDateLastYear = (Calendar) endDate.clone();
endDateLastYear.add(Calendar.YEAR, -1);
for (int i = 0; i < 31; i++) {
exitDate.add(TickingDateUtils.format(endDate.getTime()));
exitDate.add(TickingDateUtils.format(endDateLastYear.getTime()));
endDate.add(Calendar.DAY_OF_MONTH, -1);
endDateLastYear.add(Calendar.DAY_OF_MONTH, -1);
}
List<String> filteredDate = totalDate.stream()
.filter(date -> !exitDate.contains(date))
.collect(Collectors.toList());
// TODO 使用这种方法删除会报错，暂时使用mongoTemplate进行删除
// checkTicketTodayRepository.deleteByCheckticketdateIn(filteredDate);
Query query = new Query();
query.addCriteria(Criteria.where("sddate").in(filteredDate));
mongoTemplate.remove(query, "sale_data_today");
return filteredDate;
}
}