抽数删除任务

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/controller/admin/ticketing/TicketingController.java

@@ -3,6 +3,8 @@ package cn.iocoder.yudao.module.infra.controller.admin.ticketing;
 import cn.iocoder.yudao.framework.common.pojo.CommonResult;
 import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
 import cn.iocoder.yudao.module.infra.job.ticketing.TicketDataMigration;
+import cn.iocoder.yudao.module.infra.service.checkticket.CheckTicketService;
+import cn.iocoder.yudao.module.infra.service.saledata.SaleDataService;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
@@ -10,6 +12,7 @@ import org.springframework.web.bind.annotation.*;
 import javax.annotation.Resource;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.List;
 
 /**
  * @Description 测试抽数接口
@@ -22,6 +25,12 @@ public class TicketingController {
     @Resource
     TicketDataMigration ticketDataMigration;
 
+    @Resource
+    CheckTicketService checkTicketService;
+
+    @Resource
+    SaleDataService saleDataService;
+
     @GetMapping
     public CommonResult<String> testTicketing(@RequestParam("startDate") String startDate, @RequestParam("endDate")String endDate, @RequestParam("method") String method) throws Exception {
         Calendar startCalendar = Calendar.getInstance();
@@ -30,4 +39,12 @@ public class TicketingController {
         endCalendar.setTime(TickingDateUtils.parse(endDate));
         return CommonResult.success(ticketDataMigration.getTicketDataByTimeZonesToMongo(startCalendar, endCalendar, method));
     }
+    @GetMapping("/test")
+    public CommonResult<List<String>> testAll(){
+        return CommonResult.success(checkTicketService.deleteTwoYearLastMonthAgo());
+    }
+    @GetMapping("/testa")
+    public CommonResult<List<String>> testAlla(){
+        return CommonResult.success(saleDataService.deleteTwoYearLastMonthAgo());
+    }
 }

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mongodb/checkticket/CheckTicketTodayRepository.java

@@ -3,6 +3,7 @@ package cn.iocoder.yudao.module.infra.dal.mongodb.checkticket;
 
 import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket;
 import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketToday;
+import org.springframework.data.mongodb.repository.Aggregation;
 import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.stereotype.Repository;
@@ -27,4 +28,20 @@ public interface CheckTicketTodayRepository extends MongoRepository<CheckTicketT
      */
     @Query("{ 'checkticketdate': { $not: { $gte: ?0, $lt: ?1 } } }")
     List<CheckTicketToday> findNotInTimeRange(String startDate, String endDate);
+    /**
+     * 查询所有去重的 checkticketdate 字段值
+     * @return 去重后的 checkticketdate 字段值列表
+     */
+    @Aggregation(pipeline = {
+            "{$group: {_id: '$checkticketdate'}}",
+            "{$project: {_id: 0, checkticketdate: '$_id'}}"
+    })
+    List<String> findDistinctDate();
+
+    /**
+     * 删除 checkticketdate 字段值在给定列表中的所有记录
+     * @param dates 日期列表
+     */
+    @Query("{ 'checkticketdate': { $in: ?0 } }")
+    void deleteByCheckticketdateIn(List<String> dates);
 }

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mongodb/saledata/SaleDataTodayRepository.java

@@ -78,4 +78,13 @@ public interface SaleDataTodayRepository extends MongoRepository<SaleDataToday,S
      */
     @Query("{ 'sddate': { $not: { $gte: ?0, $lt: ?1 } } }")
     List<SaleDataToday> findNotInTimeRange(String startDate, String endDate);
+    /**
+     * 查询所有去重的 sddate 字段值
+     * @return 去重后的 sddate 字段值列表
+     */
+    @Aggregation(pipeline = {
+            "{$group: {_id: '$sddate'}}",
+            "{$project: {_id: 0, sddate: '$_id'}}"
+    })
+    List<String> findDistinctDate();
 }

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/job/ticketing/TicketDataMigration.java

@@ -309,35 +309,23 @@ public class TicketDataMigration {
     @XxlJob("deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow")
     public void deleteTwoYearOneMonthAgoAndUpdateLastYearTomorrow(){
         XxlJobHelper.log("删除多余数据开始...");
-        // 检票数据今年多余删除
+        // 检票数据今年去年多余删除
-        Calendar today = Calendar.getInstance();
+        XxlJobHelper.log("删除检票日期为{}", checkTicketService.deleteTwoYearLastMonthAgo().toString());
-        Calendar thirtyDaysAgo = (Calendar) today.clone(); // 克隆当前日期
+        // 售票数据今年去年多余删除
-        thirtyDaysAgo.add(Calendar.DAY_OF_MONTH, -31);
+        XxlJobHelper.log("删除售票日期为{}", saleDataService.deleteTwoYearLastMonthAgo().toString());
-        List<String> thisYearCheckTicketList = checkTicketService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(thirtyDaysAgo.getTime()), TickingDateUtils.format(today.getTime()));
-        XxlJobHelper.log("删除今年检票日期为{}", thisYearCheckTicketList.toString());
-        // 检票数据去年多余删除
-        Calendar lastYearToday = (Calendar) today.clone();
-        lastYearToday.add(Calendar.YEAR, -1);
-        Calendar lastYearThirtyDaysAgo = (Calendar) lastYearToday.clone();
-        lastYearThirtyDaysAgo.add(Calendar.DAY_OF_MONTH, -31);
-        List<String> lastYearCheckTicketList = checkTicketService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(lastYearThirtyDaysAgo.getTime()), TickingDateUtils.format(lastYearToday.getTime()));
-        XxlJobHelper.log("删除去年检票日期为{}", lastYearCheckTicketList.toString());
-        // 售票数据今年多余删除
-        List<String> thisYearSaleDataList = saleDataService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(thirtyDaysAgo.getTime()), TickingDateUtils.format(today.getTime()));
-        XxlJobHelper.log("删除今年售票日期为{}", thisYearSaleDataList.toString());
-        // 售票数据去年多余删除
-        List<String> lastYearSaleDataList = checkTicketService.deleteNotInTimeZonesByMongoDB(TickingDateUtils.format(lastYearThirtyDaysAgo.getTime()), TickingDateUtils.format(lastYearToday.getTime()));
-        XxlJobHelper.log("删除去年检票日期为{}", lastYearSaleDataList.toString());
         // 抽取去年明天的检票和售票数据
-        Calendar lastYearTomorrow = (Calendar) lastYearToday.clone();
+        Calendar lastYearTomorrow = Calendar.getInstance();
         lastYearTomorrow.add(Calendar.DAY_OF_MONTH, 1);
+        lastYearTomorrow.add(Calendar.YEAR, -1);
         try {
             this.getTicketDataByTimeZonesToMongo(lastYearTomorrow, lastYearTomorrow, "checkTicket");
+            XxlJobHelper.log("抽取去年明天的检票数据成功，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
         } catch (Exception e) {
             XxlJobHelper.log("抽取去年明天的检票数据失败，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
         }
         try {
             this.getTicketDataByTimeZonesToMongo(lastYearTomorrow, lastYearTomorrow, "saleData");
+            XxlJobHelper.log("抽取去年明天的售票数据成功，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
         } catch (Exception e) {
             XxlJobHelper.log("抽取去年明天的售票数据失败，日期为{}", TickingDateUtils.format(lastYearTomorrow.getTime()));
         }

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketService.java

@@ -47,4 +47,13 @@ public interface CheckTicketService {
      * @return java.util.List<java.lang.String> 删除的日期
      */
     List<String> deleteNotInTimeZonesByMongoDB(String startDate,String endDate);
+
+    /**
+     * 删除两年上个月以前的数据(mongodb)
+     * @param
+     * @return java.util.List<java.lang.String>
+     */
+    List<String> deleteTwoYearLastMonthAgo();
+
+    List<String> findDistinctDate();
 }

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/checkticket/CheckTicketServiceImpl.java

@@ -1,6 +1,7 @@
 package cn.iocoder.yudao.module.infra.service.checkticket;
 
 import cn.iocoder.yudao.framework.common.pojo.PageResult;
+import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
 import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
 import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
 import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket;
@@ -14,6 +15,7 @@ import com.baomidou.dynamic.datasource.annotation.Slave;
 import com.mongodb.MongoException;
 import org.springframework.data.domain.Example;
 import org.springframework.data.domain.ExampleMatcher;
+import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.stereotype.Service;
@@ -42,6 +44,8 @@ public class CheckTicketServiceImpl implements CheckTicketService {
     private CheckTicketRepository checkTicketRepository;
     @Resource
     private CheckTicketTodayRepository checkTicketTodayRepository;
+    @Resource
+    private MongoTemplate mongoTemplate;
 
     @Override
     public Long createCheckTicket(CheckTicketDO checkTicketDO) {
@@ -116,4 +120,33 @@ public class CheckTicketServiceImpl implements CheckTicketService {
         return deletedDates;
     }
 
+    @Override
+    public List<String> deleteTwoYearLastMonthAgo() {
+        List<String> totalDate = checkTicketTodayRepository.findDistinctDate();
+        List<String> exitDate = new ArrayList<>();
+        Calendar endDate = Calendar.getInstance();
+        Calendar endDateLastYear = (Calendar) endDate.clone();
+        endDateLastYear.add(Calendar.YEAR, -1);
+        for (int i = 0; i < 31; i++) {
+            exitDate.add(TickingDateUtils.format(endDate.getTime()));
+            exitDate.add(TickingDateUtils.format(endDateLastYear.getTime()));
+            endDate.add(Calendar.DAY_OF_MONTH, -1);
+            endDateLastYear.add(Calendar.DAY_OF_MONTH, -1);
+        }
+        List<String> filteredDate = totalDate.stream()
+                .filter(date -> !exitDate.contains(date))
+                .collect(Collectors.toList());
+        // TODO 使用这种方法删除会报错，暂时使用mongoTemplate进行删除
+//        checkTicketTodayRepository.deleteByCheckticketdateIn(filteredDate);
+        Query query = new Query();
+        query.addCriteria(Criteria.where("sddate").in(filteredDate));
+        mongoTemplate.remove(query, "sale_data_today");
+        return filteredDate;
+    }
+
+    @Override
+    public List<String> findDistinctDate() {
+        return checkTicketTodayRepository.findDistinctDate();
+    }
+
 }

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataService.java

@@ -49,4 +49,10 @@ public interface SaleDataService {
      * @return java.util.List<java.lang.String> 删除的日期
      */
     List<String> deleteNotInTimeZonesByMongoDB(String startDate,String endDate);
+    /**
+     * 删除两年上个月以前的数据(mongodb)
+     * @param
+     * @return java.util.List<java.lang.String>
+     */
+    List<String> deleteTwoYearLastMonthAgo();
 }

yudao-module-infra/yudao-module-infra-biz/src/main/java/cn/iocoder/yudao/module/infra/service/saledata/SaleDataServiceImpl.java

@@ -1,5 +1,6 @@
 package cn.iocoder.yudao.module.infra.service.saledata;
 
+import cn.iocoder.yudao.framework.common.util.date.TickingDateUtils;
 import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicket;
 import cn.iocoder.yudao.module.infra.dal.dataobject.checkticket.CheckTicketToday;
 import cn.iocoder.yudao.module.infra.dal.dataobject.saledata.SaleData;
@@ -9,6 +10,9 @@ import cn.iocoder.yudao.module.infra.dal.mongodb.saledata.SaleDataRepository;
 import cn.iocoder.yudao.module.infra.dal.mongodb.saledata.SaleDataTodayRepository;
 import cn.iocoder.yudao.module.infra.dal.mysql.saledata.SaleDataMapper;
 import com.mongodb.MongoException;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.stereotype.Service;
 import org.springframework.validation.annotation.Validated;
 
@@ -34,6 +38,9 @@ public class SaleDataServiceImpl implements SaleDataService {
     @Resource
     private SaleDataMapper saleDataMapper;
 
+    @Resource
+    private MongoTemplate mongoTemplate;
+
     @Override
     public Long createSaleData(SaleDataDO saleDataDO) {
         // 插入
@@ -106,5 +113,27 @@ public class SaleDataServiceImpl implements SaleDataService {
                 .collect(Collectors.toList());
         return deletedDates;
     }
-
+    @Override
+    public List<String> deleteTwoYearLastMonthAgo() {
+        List<String> totalDate = saleDataTodayRepository.findDistinctDate();
+        List<String> exitDate = new ArrayList<>();
+        Calendar endDate = Calendar.getInstance();
+        Calendar endDateLastYear = (Calendar) endDate.clone();
+        endDateLastYear.add(Calendar.YEAR, -1);
+        for (int i = 0; i < 31; i++) {
+            exitDate.add(TickingDateUtils.format(endDate.getTime()));
+            exitDate.add(TickingDateUtils.format(endDateLastYear.getTime()));
+            endDate.add(Calendar.DAY_OF_MONTH, -1);
+            endDateLastYear.add(Calendar.DAY_OF_MONTH, -1);
+        }
+        List<String> filteredDate = totalDate.stream()
+                .filter(date -> !exitDate.contains(date))
+                .collect(Collectors.toList());
+        // TODO 使用这种方法删除会报错，暂时使用mongoTemplate进行删除
+//        checkTicketTodayRepository.deleteByCheckticketdateIn(filteredDate);
+        Query query = new Query();
+        query.addCriteria(Criteria.where("sddate").in(filteredDate));
+        mongoTemplate.remove(query, "sale_data_today");
+        return filteredDate;
+    }
 }