java一个“批处理”工具类

前言

我们在日常的开发中有时候会面临着大批量的执行某个方法的时候，我们有时候会需要把一个大的任务拆分成小的任务执行，这个时候我提供的工具就派上用场了

下面的工具支持同步的拆分执行和通过线程池异步执行.

本工具充分利用了java的泛型和lambda表达式， 可以执行各种批量的业务逻辑，但是使用异步方式的时候要注意线程安全的问题。

使用方式

同步方法的使用方式
这个是批量的从数据库中查询出数据的使用方式

 List<DialBookDO> dialPlanPhoneBookDOList = BatchUtil.batchApply(phones,
  ps -> {
         List<DialPlanDO> phoneBookDOS = dialPlankMapper.selectByPhones(planId, shopCode, ps);
         return phoneBookDOS;
  });

异步的使用方式

 List<DialBookDO> dialPlanPhoneBookDOList = BatchUtil.batchApplyAsync(phones,
  ps -> {
         List<DialPlanDO> phoneBookDOS = dialPlankMapper.selectByPhones(planId, shopCode, ps);
         return phoneBookDOS;
  });

源码

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.*;
import java.util.function.Function;
import java.util.stream.Collectors;

/**
 * @Description 批量执行工具
 * @Author xiezc
 * @Version 1.0
 * @Date 2020/8/26 2:35 下午
 **/
public class BatchUtil {

    /**
     * 分批次处理的每批次的大小
     */
    public static int batchSize = 200;

    /**
     * 异步执行任务的线程池
     */
    public static ExecutorService executorService =
            new ThreadPoolExecutor(0, 10, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(2048), new ThreadFactory() {
                @Override
                public Thread newThread(Runnable r) {
                    int i = (int) (Math.random() * 100);
                    return new Thread(r, "BatchUtil-executorService-" + i);
                }
            }, new ThreadPoolExecutor.CallerRunsPolicy());

    /**
     * 同步分批次处理
     *
     * @param collect
     * @param function
     * @param <T>
     * @param <R>
     * @return
     */
    public static <T, R> List<R> batchApply(List<T> collect, Function<List<T>, List<R>> function) {
        return batchApply(collect, function, batchSize);
    }

    /**
     * 异步分批次处理
     *
     * @param collect
     * @param function
     * @param <T>
     * @param <R>
     * @return
     */
    public static <T, R> List<R> batchApplyAsync(List<T> collect, Function<List<T>, List<R>> function) {
        return batchApplyAsync(collect, function, batchSize);
    }

    /**
     * 同步分批次处理
     *
     * @param collect
     * @param function
     * @param batchSize
     * @param <T>
     * @param <R>
     * @return
     */
    public static <T, R> List<R> batchApply(List<T> collect, Function<List<T>, List<R>> function, int batchSize) {
        if (batchSize <= 0) {
            batchSize = BatchUtil.batchSize;
        }
        if (collect.size() < batchSize) {
            return function.apply(collect);
        } else {
            List<List<R>> lists = applySync(collect, function, batchSize);
            return lists.stream()
                    .flatMap(Collection::stream)
                    .collect(Collectors.toList());
        }
    }

    /**
     * 异步分批次处理
     *
     * @param collect
     * @param function
     * @param batchSize
     * @param <T>
     * @param <R>
     * @return
     */
    public static <T, R> List<R> batchApplyAsync(List<T> collect, Function<List<T>, List<R>> function, int batchSize) {
        if (batchSize <= 0) {
            batchSize = BatchUtil.batchSize;
        }
        if (collect.size() < batchSize) {
            return function.apply(collect);
        } else {

            List<Future<List<R>>> futures = applyAsync(collect, function, batchSize);

            List<R> results = new ArrayList<>();
            for (Future<List<R>> future : futures) {
                try {
                    results.addAll(future.get());
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
            return results;
        }
    }


    private static <T, R> List<Future<List<R>>> applyAsync(List<T> collect, Function<List<T>, List<R>> function, int batchSize) {
        int size = collect.size();
        int start = 0;
        List<Future<List<R>>> futures = new ArrayList<>();
        while (start < size) {
            int end = start + batchSize;
            if (end > size) {
                end = size;
            }
            List<T> ts = collect.subList(start, end);
            Future<List<R>> submit = executorService.submit(() -> {
                List<R> apply = function.apply(ts);
                return apply;
            });
            futures.add(submit);
            start = start + batchSize;
        }
        return futures;
    }


    private static <T, R> List<List<R>> applySync(List<T> collect, Function<List<T>, List<R>> function, int batchSize) {
        int size = collect.size();
        int start = 0;
        List<List<R>> futures = new ArrayList<>();
        while (start < size) {
            int end = start + batchSize;
            if (end > size) {
                end = size;
            }
            List<T> ts = collect.subList(start, end);
            List<R> apply = function.apply(ts);
            futures.add(apply);
            start = start + batchSize;
        }
        return futures;
    }
}