Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

1.临时文件保留日期可通过配置文件修改 2.对fixed线程池状态监控 #2184

Open
wants to merge 2 commits into
base: dev-0.3
Choose a base branch
from
Open

1.临时文件保留日期可通过配置文件修改 2.对fixed线程池状态监控 #2184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1e6bfc5
1.临时文件保留日期可通过配置修改，2.以及进行fixed线程池状态监控
wangshuai67 Apr 1, 2021
e601ae9
1.修改定时任务配置
wangshuai67 Apr 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions server/src/main/java/edp/core/utils/DateUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,19 @@ public static String getTheDayBeforeAWeekYYYYMMDD() {
return formatter.format(calendar.getTime());
}

/**
* 文件保留天数可配置
* @param amount
* @return
*/
public static String getTheDayBeforeAWeekYYYYMMDD(int amount) {
SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
Calendar calendar = Calendar.getInstance();
calendar.setTime(currentDate());
calendar.add(Calendar.DAY_OF_MONTH, amount);

return formatter.format(calendar.getTime());
}
public static String getNowDateFormatCustom(String format) {
SimpleDateFormat formatter = new SimpleDateFormat(format);

Expand Down
32 changes: 31 additions & 1 deletion server/src/main/java/edp/core/utils/FileUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import edp.davinci.core.enums.FileTypeEnum;
import edp.davinci.core.enums.LogNameEnum;
import edp.davinci.service.excel.MsgWrapper;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
Expand All @@ -38,10 +40,13 @@
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.*;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.file.*;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.zip.ZipEntry;
Expand All @@ -51,6 +56,7 @@


@Component
@Slf4j
public class FileUtils {

private static final Logger scheduleLogger = LoggerFactory.getLogger(LogNameEnum.BUSINESS_SCHEDULE.getName());
Expand Down Expand Up @@ -153,6 +159,7 @@ public void download(String filePath, HttpServletResponse response) {
response.reset();
response.addHeader("Content-Disposition", "attachment;filename=" + new String(file.getName().getBytes(), "UTF-8"));
response.addHeader("Content-Length", EMPTY + file.length());
// todo 通过ZipOutputStream压缩后传输
os = new BufferedOutputStream(response.getOutputStream());
response.setContentType("application/octet-stream;charset=UTF-8");
os.write(buffer);
Expand Down Expand Up @@ -194,7 +201,12 @@ public boolean remove(String filePath) {
*/
public static void deleteDir(File dir) {

if (dir.isFile() || dir.list().length == 0) {
if(!dir.exists()){
// 减少因手动删除文件导致的日志错误报警
log.info("文件路径不存在或已被删除dir:[{}]",dir.getPath());
return;
}
if(dir.isFile() || dir.list().length == 0) {
dir.delete();
}
else {
Expand All @@ -204,7 +216,25 @@ public static void deleteDir(File dir) {
dir.delete();
}
}
/**
* NIO删除文件夹及其下文件
*
* @param dir
* @return
*/
public static void deleteDirByNio(File dir) throws IOException {

if (dir.isFile() || dir.list().length == 0) {
Path path = Paths.get(dir.getPath());
Files.deleteIfExists(path);
}
else {
for (File f : dir.listFiles()) {
deleteDirByNio(f);
}
dir.delete();
}
}
/**
* 格式化文件目录
*
Expand Down
69 changes: 56 additions & 13 deletions server/src/main/java/edp/davinci/schedule/SystemSchedule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,17 @@
import edp.davinci.model.ShareDownloadRecord;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.io.File;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;

@Slf4j
@Component
Expand All @@ -55,24 +59,37 @@ public class SystemSchedule {

@Autowired
private ShareDownloadRecordMapper shareDownloadRecordMapper;
@Value("${file.temp.remain.days:7}")
private int fileTempRemainDays;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

临时文件保留日期如果是个没有层次的属性，建议使用file_temp_remain_days

@Value("${thread.pool.health.check.enable:true}")
private boolean threadPoolHealthCheckEnable;

private static final int THREAD_POOL_QUEUE_ALARM_SIZE=2000;
private static final ExecutorService CLEAR_TEMPDIR_THREADPOOL = Executors.newFixedThreadPool(3);

@Scheduled(cron = "0 0 1 * * *")
public void clearTempDir() {

//下载内容文件保留7天，记录保留1月
String downloadDir = fileUtils.fileBasePath + Consts.DIR_DOWNLOAD + DateUtils.getTheDayBeforeAWeekYYYYMMDD();
String tempDir = fileUtils.fileBasePath + Consts.DIR_TEMP + DateUtils.getTheDayBeforeNowDateYYYYMMDD();
String csvDir = fileUtils.fileBasePath + File.separator + FileTypeEnum.CSV.getType();

final String download = fileUtils.formatFilePath(downloadDir);
final String temp = fileUtils.formatFilePath(tempDir);
final String csv = fileUtils.formatFilePath(csvDir);

CLEAR_TEMPDIR_THREADPOOL.execute(() -> FileUtils.deleteDir(new File(download)));
CLEAR_TEMPDIR_THREADPOOL.execute(() -> FileUtils.deleteDir(new File(temp)));
CLEAR_TEMPDIR_THREADPOOL.execute(() -> FileUtils.deleteDir(new File(csv)));
checkThreadPoolExecutorStatus();
// 下载内容文件默认保留7天，记录保留1月，在月初和月末大数据量导出情况下部分文件删除IO异常失败，导致磁盘空间撑满
// 可自定义配置文件保留日期配置文件中file.temp.remain.days=n 运维可根据集团实际情况进行配置
String downloadDir=null ;
String tempDir=null ;
try {
log.debug("下载内容文件默认保留[{}]天,可在配置文件通过file.temp.remain.days=n进行配置",fileTempRemainDays);
downloadDir = fileUtils.fileBasePath + Consts.DIR_DOWNLOAD + DateUtils.getTheDayBeforeAWeekYYYYMMDD(fileTempRemainDays);
tempDir = fileUtils.fileBasePath + Consts.DIR_TEMP + DateUtils.getTheDayBeforeNowDateYYYYMMDD();
String csvDir = fileUtils.fileBasePath + File.separator + FileTypeEnum.CSV.getType();

final String download = fileUtils.formatFilePath(downloadDir);
final String temp = fileUtils.formatFilePath(tempDir);
final String csv = fileUtils.formatFilePath(csvDir);
log.info("downloadDir:{},tempDir:{},csv:{}",download,temp,csv);
CLEAR_TEMPDIR_THREADPOOL.execute(() -> FileUtils.deleteDir(new File(download)));
CLEAR_TEMPDIR_THREADPOOL.execute(() -> FileUtils.deleteDir(new File(temp)));
CLEAR_TEMPDIR_THREADPOOL.execute(() -> FileUtils.deleteDir(new File(csv)));
} catch (Exception e) {
log.error("文件删除定时任务执行异常downloadDir:[{}]，tempDir:[{}],请检查IO以及手动处理临时文件",downloadDir,tempDir);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

建议也一并输出下csv的目录，'，'和','需要统一

}
}

@Scheduled(cron = "0 0/2 * * * *")
Expand Down Expand Up @@ -100,6 +117,32 @@ public void clearShareDownloadRecord() {
shareDownloadRecordMapper.deleteByCondition();
}

/**
* 线程池状态健康检查
* 可通过 thread.pool.health.check.enable=false 关闭监控检查
* @param
*/
@Scheduled(cron = "${thread.pool.health.check.cron:0 0 1/1 * * *}")
public void checkThreadPoolExecutorStatus(){
if(!threadPoolHealthCheckEnable){
return;
}
ThreadPoolExecutor tpe = ((ThreadPoolExecutor) CLEAR_TEMPDIR_THREADPOOL);
int queueSize = tpe.getQueue().size();
int activeCount = tpe.getActiveCount();
long completedTaskCount = tpe.getCompletedTaskCount();
long taskCount = tpe.getTaskCount();
BlockingQueue<Runnable> queue = tpe.getQueue();
if(queue.size()>THREAD_POOL_QUEUE_ALARM_SIZE){
log.warn("当前排队线程数[{}]，当前活动线程数[{}]，执行完成线程数[{}]，总线程数[{}]", queueSize, activeCount, completedTaskCount, taskCount);
}
if(queue.size()>THREAD_POOL_QUEUE_ALARM_SIZE*2){
log.error("线程执行效率异常，可能存在IO异常，当前排队线程数[{}]，当前活动线程数[{}]，执行完成线程数[{}]，总线程数[{}]", queueSize, activeCount, completedTaskCount, taskCount);
}
else{
log.info("当前排队线程数[{}]，当前活动线程数[{}]，执行完成线程数[{}]，总线程数[{}]", queueSize, activeCount, completedTaskCount, taskCount);
}
}
private void deleteFile(File file){
if(file == null || !file.exists()){
return;
Expand Down