Skip to content

T
test-data-manalysis
Commit 275a9c44 by zhugs
Options

init

parents
正在显示 包含 670 行增加 和 0 行删除
.gitignore 0 → 100644
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
logs
### IntelliJ IDEA ###
.idea
.idea/modules.xml
.idea/jarRepositories.xml
.idea/compiler.xml
.idea/libraries/
*.iws
*.iml
*.ipr
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store
\ No newline at end of file
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhugso</groupId>
<artifactId>test-data</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>25</maven.compiler.source>
<maven.compiler.target>25</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>5.4.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.4.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.17</version>
</dependency>
<!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-classic -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.5.18</version>
<scope>compile</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>7.0.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.mysql/mysql-connector-j -->
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<version>9.4.0</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.40</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.20.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
src/main/java/Main.java 0 → 100644
import database.InsertData;
import lombok.extern.slf4j.Slf4j;
import text.TextFileWatcher;
import util.ConfigUtil;
import java.io.File;
import java.nio.file.Path;
import java.util.function.Predicate;
@Slf4j
public class Main {
public static void main(String[] args) {
Path dirText = Path.of(ConfigUtil.getWatchDir());
// 设置要匹配的文件规则
Predicate<File> textFilter = file -> {
String name = file.getName();
if (!(name.contains("PASS") && name.endsWith(".txt")))
return false;
String fileName = new InsertData().getFileName(name);
if (name.equals(fileName)) {
log.info("文件重复，跳过: {}", name);
return false;
}
return true;
};
TextFileWatcher textWatcher = new TextFileWatcher(dirText, textFilter);
textWatcher.startWatch();
// 监听 excel log文件
// File dirExcel = new File("C:\\Users\\zhugso\\Projects\\chenyang\\testdata\\exc");
// Predicate<File> excelFilter = file -> {
// String name = file.getName();
// return !name.contains("$");
// };
//
// ExcelFileWatcher excelWatcher = new ExcelFileWatcher(dirExcel, excelFilter);
// excelWatcher.startWatch();
// 阻塞主线程
try {
Thread.currentThread().join();
} catch (InterruptedException e) {
log.error(e.getMessage(), e);
}
}
}
src/main/java/database/DataSource.java 0 → 100644
package database;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import util.ConfigUtil;
import java.sql.Connection;
import java.sql.SQLException;
public class DataSource {
private static final HikariConfig config = new HikariConfig();
private static final HikariDataSource ds;
static {
config.setJdbcUrl(ConfigUtil.getDBJdbcUrl());
config.setUsername(ConfigUtil.getDBUsername());
config.setPassword(ConfigUtil.getDBPassword());
config.addDataSourceProperty("cachePrepStmts", "true");
config.addDataSourceProperty("prepStmtCacheSize", "250");
config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048");
ds = new HikariDataSource(config);
}
private DataSource() {
}
public static Connection getConnection() throws SQLException {
return ds.getConnection();
}
}
src/main/java/database/InsertData.java 0 → 100644
package database;
import domain.ResultData;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
public class InsertData {
//sql
private String insertFileAttr = """
INSERT INTO file_attr (type, file_name, zip_file_url, file_url, sn, time)
VALUES (?, ?, ?, ?, ?, ?)
""";
private String queryFileAttr = """
SELECT id FROM file_attr
WHERE file_name=?
""";
private String insertLogDetails = """
INSERT INTO log_details (file_id, group_name, item_name, item_value, item_spec)
VALUES (?, ?, ?, ?, ?)
""";
private String queryFileNames = """
SELECT file_name FROM file_attr
""";
private String queryFileName = """
SELECT file_name FROM file_attr WHERE file_name=?
""";
public void insert(ResultData rs) {
try (Connection con = DataSource.getConnection()) {
try (PreparedStatement ps = con.prepareStatement(insertFileAttr)) {
ps.setString(1, rs.getType());
ps.setString(2, rs.getFileName());
ps.setString(3, rs.getZipFileUrl());
ps.setString(4, rs.getFileUrl());
ps.setString(5, rs.getSn());
ps.setDate(6, new Date(rs.getTime().getTime()));
ps.execute();
}
int id;
try (PreparedStatement ps = con.prepareStatement(queryFileAttr)) {
ps.setString(1, rs.getFileName());
ResultSet resultSet = ps.executeQuery();
resultSet.next();
id = resultSet.getInt(1);
}
try (PreparedStatement ps = con.prepareStatement(insertLogDetails)) {
ps.setInt(1, id);
List<ResultData.Item> items = rs.getItem();
for (ResultData.Item item : items) {
ps.setString(2, item.getGroupName());
ps.setString(3, item.getItemName());
ps.setDouble(4, item.getValue());
ps.setString(5, item.getRange());
ps.execute();
}
}
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
public String getFileName(String fileName) {
try (Connection con = DataSource.getConnection()) {
try (PreparedStatement ps = con.prepareStatement(queryFileName)) {
ps.setString(1, fileName);
ResultSet resultSet = ps.executeQuery();
if (resultSet.next()) {
return resultSet.getString(1);
}
return "";
}
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
public List<String> getFileNames() {
List<String> list = new ArrayList<>();
try (Connection con = DataSource.getConnection()) {
try (PreparedStatement ps = con.prepareStatement(queryFileNames)) {
ResultSet resultSet = ps.executeQuery();
while (resultSet.next()) {
list.add(resultSet.getString(1));
}
return list;
}
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
}
src/main/java/domain/ResultData.java 0 → 100644
package domain;
import lombok.Data;
import java.util.Date;
import java.util.List;
@Data
public class ResultData {
private String type;
private String fileName;
private String zipFileUrl;
private String fileUrl;
private String sn;
private Date time;
private List<Item> item;
@Data
public static class Item {
private String groupName;
private String itemName;
private String unit;
private Double value;
private String range;
}
}
src/main/java/text/TextFileWatcher.java 0 → 100644
package text;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.*;
import java.util.function.Predicate;
@Slf4j
public class TextFileWatcher {
private final Path dir;
private final Predicate<File> fileFilter;
public TextFileWatcher(Path dir, Predicate<File> fileFilter) {
this.dir = dir;
this.fileFilter = fileFilter;
}
public void startWatch() {
Thread.ofVirtual().start(this::watchLoop);
}
private void watchLoop() {
try {
// 初始处理，将文件夹下有的文件先处理
File[] files = dir.toFile().listFiles(fileFilter::test);
if (files != null) {
for (File f : files) {
TextHandler textHandler = new TextHandler(f);
Thread.ofVirtual().start(textHandler);
}
}
WatchService watchService = FileSystems.getDefault().newWatchService();
dir.register(watchService, StandardWatchEventKinds.ENTRY_CREATE);
while (true) {
WatchKey key = watchService.take();
for (WatchEvent<?> event : key.pollEvents()) {
Path fullPath = dir.resolve((Path) event.context());
File file = fullPath.toFile();
if(fileFilter.test(file) && waitForFileReady(file, 3, 500)){
log.info("文件已创建: {}", file.getName());
TextHandler textHandler = new TextHandler(file);
Thread.ofVirtual().start(textHandler);
}
}
key.reset();
}
} catch (IOException | InterruptedException e) {
log.error(e.getMessage());
}
}
// 避免文件正在写入，重试机制
private boolean waitForFileReady(File file, int maxRetries, long intervalMs) {
for (int i = 0; i < maxRetries; i++) {
try (FileInputStream fis = new FileInputStream(file)) {
// 能打开就认为文件已写入完成
return true;
} catch (IOException e) {
try {
Thread.sleep(intervalMs);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
break;
}
}
}
return false; // 超过最大重试次数仍无法读取
}
}
src/main/java/text/TextHandler.java 0 → 100644
package text;
import database.InsertData;
import domain.ResultData;
import lombok.extern.slf4j.Slf4j;
import java.io.*;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Slf4j
public class TextHandler implements Runnable {
private final File file;
public TextHandler(File file) {
this.file = file;
}
@Override
public void run() {
log.info("文件处理:{}", file.getName());
String begin = "=+ SUMMARY begin =+";
String end = "=+ SUMMARY end {2}=+";
Pattern patternBegin = Pattern.compile(begin);
Pattern patternEnd = Pattern.compile(end);
boolean beginFlag = false;
List<String> patternText = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
String line;
while ((line = reader.readLine()) != null) {
if (line.isEmpty())
continue;
if (patternBegin.matcher(line).find() && !beginFlag) {
beginFlag = true;
continue;
}
if (patternEnd.matcher(line).find() && beginFlag) {
break;
}
if (beginFlag) {
patternText.add(line);
}
}
} catch (IOException e) {
log.error(e.getMessage());
}
// 目标数据匹配
ResultData data = StringParser(patternText);
data.setFileName(file.getName());
String[] ss = file.getName().split("_");
data.setSn(ss[1]);
DateFormat format = new SimpleDateFormat("yyyyMMdd-HHmmss");
Date parse;
try {
parse = format.parse(ss[5]);
} catch (ParseException e) {
throw new RuntimeException(e);
}
data.setTime(parse);
data.setType("Wifi");
// 压缩日志文件并删除
String zipName = zipLogFile(file);
data.setZipFileUrl(zipName);
// data.setFileUrl(file.getAbsolutePath());
new InsertData().insert(data);
}
private ResultData StringParser(List<String> patternText) {
String regex = "\\[PASS] (?<itemName>.+?) (?<unit>\\w+), value = \"(?<value>[-\\d.]+)\" (?<range>\\([^)]*\\))";
Pattern pattern = Pattern.compile(regex);
ResultData data = new ResultData();
List<ResultData.Item> items = new ArrayList<>();
for (int i = 10; i < patternText.size(); i += 3) {
String s1 = patternText.get(i - 2);
String groupName = s1.substring(4);
for (int j = i - 1; j < i + 1; j++) {
String sLine = patternText.get(j);
Matcher matcher = pattern.matcher(sLine);
if (matcher.find()) {
ResultData.Item item = new ResultData.Item();
item.setGroupName(groupName);
item.setItemName(matcher.group("itemName"));
item.setUnit(matcher.group("unit"));
item.setValue(Double.valueOf(matcher.group("value")));
item.setRange(matcher.group("range"));
items.add(item);
}
}
}
data.setItem(items);
return data;
}
private String zipLogFile(File file) {
String zipFile = file.getParent() + File.separator
+ getBaseName(file.getName()) + ".zip";
try (FileOutputStream fos = new FileOutputStream(zipFile);
ZipOutputStream zos = new ZipOutputStream(fos)) {
try (FileInputStream fis = new FileInputStream(file)) {
// 创建 ZipEntry，名字就是压缩包里的文件名
ZipEntry zipEntry = new ZipEntry(file.getName());
zos.putNextEntry(zipEntry);
byte[] buffer = new byte[1024];
int length;
while ((length = fis.read(buffer)) >= 0) {
zos.write(buffer, 0, length);
}
}
log.info("压缩完成:{}", zipFile);
log.info("文件是否删除: {}, {}", file.delete(), file.getName());
} catch (IOException e) {
log.error(e.getMessage());
}
return zipFile;
}
private String getBaseName(String fileName) {
int dotIndex = fileName.lastIndexOf('.');
return (dotIndex == -1) ? fileName : fileName.substring(0, dotIndex);
}
}
src/main/java/util/ConfigUtil.java 0 → 100644
package util;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
public class ConfigUtil {
private static final Properties props = new Properties();
private static final String propFilePath = "/config.properties";
static {
try (InputStream is = ConfigUtil.class.getResourceAsStream(propFilePath)) {
props.load(is);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private ConfigUtil() {
}
public static String getWatchDir() {
return props.getProperty("watch_dir");
}
public static String getDBJdbcUrl() {
return props.getProperty("jdbc_url");
}
public static String getDBUsername() {
return props.getProperty("username");
}
public static String getDBPassword() {
return props.getProperty("password");
}
}
src/main/java/util/ZipUtil.java 0 → 100644
package util;
import java.io.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
public class ZipUtil {
private ZipUtil() {
}
//String content = readSingleTextFileFromZip(zipFile, "UTF-8")
public static String readSingleTextFileFromZip(File zipFile, String charset) {
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile))) {
ZipEntry entry = zis.getNextEntry();
if (entry != null && !entry.isDirectory()) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(zis, charset))) {
StringBuilder sb = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("\n");
}
return sb.toString();
}
}
} catch (IOException e) {
throw new RuntimeException("读取 zip 文件失败", e);
}
return null;
}
public static byte[] readSingleFileFromZip(File zipFile) {
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile))) {
ZipEntry entry = zis.getNextEntry();
if (entry != null && !entry.isDirectory()) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
int len;
while ((len = zis.read(buffer)) > 0) {
baos.write(buffer, 0, len);
}
return baos.toByteArray();
}
} catch (IOException e) {
throw new RuntimeException("读取 zip 文件失败", e);
}
return null;
}
}
src/main/resources/config.properties 0 → 100644
watch_dir=C:/Users/zhugso/Projects/chenyang/testdata/text
jdbc_url=jdbc:mysql://demo.docmis.cn:21702/test_data?serverTimezone=UTC%2B8&useUnicode=true&characterEncoding=utf8&rewriteBatchedStatements=true
username=root
password=Gbms123456
src/main/resources/logback.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="10 seconds">
<!-- 定义日志文件路径 -->
<property name="LOG_PATH" value="logs/app.log" />
<!-- 控制台输出 -->
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<!-- 文件输出 -->
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_PATH}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 每天生成一个新日志文件 -->
<fileNamePattern>logs/app.%d{yyyy-MM-dd}.log</fileNamePattern>
<!-- 保留最近30天的日志 -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<!-- 日志级别配置 -->
<root level="INFO">
<appender-ref ref="CONSOLE" />
<appender-ref ref="FILE" />
</root>
</configuration>
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论