Java读取资源文件及处理CSV/Excel，进程管理与高效批量操作-CSDN博客

读取 resources 目录下的文件（IDEA 中和 Jar 运行都可用）

InputStream inputStream = App.class.getClassLoader().getResourceAsStream("data.csv");
    // 将 InputStream 转换为字符串
    public static String inputStreamToString(InputStream inputStream) {
        StringBuilder stringBuilder = new StringBuilder();
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
            String line;
            while ((line = reader.readLine()) != null) {
                stringBuilder.append(line).append("\n");  // 加入换行符
            }
        } catch (IOException e) {
            log.error("异常", e);
        }
        return stringBuilder.toString();
    }

对象列表分组

    // 旧版分组逻辑（已废弃，保留供参考）
    // Map<String, List<ResultTower>> line_towerMap = new LinkedHashMap<>();
    // for (ResultTower item : towerList) {
    //     String objectId = item.get线路objectId();
    //     if (line_towerMap.containsKey(objectId)) {
    //         line_towerMap.get(objectId).add(item);
    //     } else {
    //         // 实际数据中线路最多杆塔数为310，预留空间至320
    //         List<ResultTower> resultTowerList = new ArrayList<>(320);
    //         resultTowerList.add(item);
    //         line_towerMap.put(objectId, resultTowerList);
    //     }
    // }

    // 按照线路 objectId 分组，保留线路杆塔顺序
    Map<String, List<ResultTower>> line_towerMap = towerList.stream()
            .collect(Collectors.groupingBy(
                    ResultTower::get线路objectId,               // 根据线路 objectId 分组
                    LinkedHashMap::new,               // 使用 LinkedHashMap 保证顺序
                    Collectors.toList()               // 将每个组的元素收集到 List 中
            ));

读取csv文件

添加 Maven 依赖

        <dependency>
            <groupId>com.opencsv</groupId>
            <artifactId>opencsv</artifactId>
            <version>5.7.1</version>
        </dependency>

代码示例：

    /**
     * 按行读取 CSV 文件数据
     */
    public void readCsv() {
        String file = "D:\\tmp\\20.csv";
        // 逐行读取
        try (CSVReader reader = new CSVReader(new BufferedReader(new FileReader(file)));) {
            HeaderColumnNameMappingStrategy<LineEntity> strategy = new HeaderColumnNameMappingStrategy<>();
            strategy.setType(LineEntity.class);
            CsvToBean<LineEntity> csvToBean = new CsvToBeanBuilder<LineEntity>(reader).withMappingStrategy(strategy).build();
            csvToBean.iterator().forEachRemaining(lineEntity -> {
                // TODO: 处理每行数据
                System.out.println();
            });
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

import com.opencsv.bean.CsvBindByName;
import lombok.Data;

// csv行数据映射为对象
@Data
public class LineEntity {
    @CsvBindByName(column = "主键")// csv列名
    private String id;
}

读取 Excel 文件

添加 Maven 依赖

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>easyexcel</artifactId>
            <version>3.1.1</version>
        </dependency>

代码

    /**
     * 按行读取 Excel 文件数据
     */
    public void readExcel() {
        String file = "D:\\tmp\\1.xlsx";
        EasyExcel.read(file, ExcelLineEntity.class, new ReadListener<ExcelLineEntity>() {

            @Override
            public void invoke(ExcelLineEntity data, AnalysisContext context) {
                // TODO: 处理每行数据
                System.out.println();
            }

            @Override
            public void doAfterAllAnalysed(AnalysisContext context) {
                System.out.println("Excel 所有数据读取结束");
            }

        }).sheet().doRead();
    }

import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;

// excel行数据映射为对象
@Data
public class ExcelLineEntity {
    @ExcelProperty("主键")// excel列名
    private String id;
}

查找并终止进程

# 查找指定JAR包的进程并强制终止
ps -ef | grep awesome-java-0.1.jar | grep -v grep | awk '{print $2}' | xargs kill -9

指定启动参数或非 Web 应用启动

java -jar awesome-java-0.1.jar --server.port=9999 --spring.main.web-application-type=none

瘦包

            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <!--                瘦包-->
                <dependencies>
                    <dependency>
                        <groupId>org.springframework.boot.experimental</groupId>
                        <artifactId>spring-boot-thin-layout</artifactId>
                        <version>1.0.28.RELEASE</version>
                    </dependency>
                </dependencies>
            </plugin>

下载项目所需的所有依赖：java -Dthin.dryrun=true -Dthin.root=. -jar awesome-java-0.1.jar
运行瘦包：java -Dthin.root=. -jar awesome-java-0.1.jar

分页与多线程数据处理

PageHelper 分页在大表场景下性能较差

        <!-- 选择适合当前 Spring Boot 的版本 -->
        <dependency>
            <groupId>com.github.pagehelper</groupId>
            <artifactId>pagehelper-spring-boot-starter</artifactId>
            <version>1.4.1</version>
        </dependency>

import com.github.pagehelper.PageHelper;
import lombok.extern.slf4j.Slf4j;

import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

// 批量计算
@Slf4j
public class BatchCalUtil {
    private static final int CPU_NUM = Runtime.getRuntime().availableProcessors();

    private static final ThreadPoolExecutor pool = (ThreadPoolExecutor) Executors.newFixedThreadPool(CPU_NUM);
    
    // 分页_多线程处理数据
    public String batchCal() {
        int pageSize = 10000;
        for (int i = 1; i < Integer.MAX_VALUE; i++) {
            // 分页查询, 数据量大性能很差, 必须使用 where id>? limit 1000
            PageHelper.startPage(i, pageSize);
            List<Object> pageData = select();
            if (!pageData.isEmpty()) {
                log.info("读取第{}页", i);
                for (Object data : pageData) {
                    pool.execute(() -> {
                        try {
                            // todo 处理数据
                        } catch (Exception e) {
                            log.error("异常:", e);
                        }
                    });
                }
                // 控制生产速度
                while (pool.getQueue().size() > 2 * pageSize) {
                    log.info("wait: {}, finish: {}", pool.getQueue().size(), pool.getCompletedTaskCount());
                    try {
                        TimeUnit.SECONDS.sleep(1);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    }
                }

            } else {
                log.info("处理完毕");
                break;
            }
        }
        pool.shutdown();
        return "处理完毕";
    }

    List<Object> select() {
        throw new RuntimeException();
    }
}

高效的批量插入

	// 注意控制列表大小，使每次 SQL 语句的长度不超过 `show global variables like 'max_allowed_packet';` 查询结果的值，MySQL 8 默认约为 64 MB。
    @Insert("<script> INSERT INTO `test_table`(`id`, `age`) VALUES <foreach collection=\"list\" separator=\",\" item=\"item\">  (#{item.id}, #{item.age}) </foreach> </script>")
    int insertBatch(List<Entity> list);

MinIO 工具类

import io.minio.*;
import io.minio.errors.*;
import io.minio.http.Method;

import java.io.File;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.TimeUnit;

/**
 * MinIO 工具类
 * 启动服务示例：
 * wget https://dl.minio.org.cn/server/minio/release/linux-amd64/minio
 * chmod +x minio
 * export MINIO_ROOT_USER=minioadmin
 * export MINIO_ROOT_PASSWORD=minioadmin
 * ./minio server data --address :9005
 *
 * <dependency>
 * <groupId>io.minio</groupId>
 * <artifactId>minio</artifactId>
 * <version>8.4.3</version>
 * </dependency>
 */
public class MinioUtil {

    private final static MinioClient minioClient;

    static {
        minioClient = MinioClient.builder().endpoint("http://172.18.240.55:9000").credentials("minioadmin", "minioadmin").build();
    }

    public static void main(String[] args) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
        String bucket = "testbucket";
        boolean bucketExists = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucket).build());
        if (!bucketExists) {
            // 创建存储桶
            minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucket).build());
        }
        String pathname = "D:\\e02a7f8ca1c6c8.pdf";
        File uploadFile = new File(pathname);
        String fileKey = "file1.key";
//        ObjectWriteResponse writeResponse = minioClient.putObject(PutObjectArgs.builder()
//                .bucket(bucket)
//                .object(fileKey)
//                .stream(new BufferedInputStream(new FileInputStream(uploadFile)), uploadFile.length(), 5242880L)
//                .build());
        ObjectWriteResponse writeResponse = minioClient.uploadObject(UploadObjectArgs.builder().bucket(bucket).object(fileKey).filename(pathname).build());
        System.out.println();
        String fileUrl = minioClient.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder().expiry(1, TimeUnit.DAYS).bucket(bucket).method(Method.GET).object(fileKey).build());
        System.out.println();
    }
}