Implementing a FIFO Export Queue for Large Data Exports in Java

The business requirement is to prevent performance degradation caused by simultaneous full‑data exports from a growing MySQL database, so a FIFO export queue with a fixed size is introduced to serialize export tasks and record file information.

Key domain objects include ExportQueue (a bounded queue that provides enqueue, dequeue, and status queries), user information, and an Export class that defines asynchronous export methods accessible via a UI.

Implementation of the export queue:

package com.example.system.config;

import com.example.system.api.domain.ExportUser;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import java.util.LinkedList;

@Slf4j
@Component
public class ExportQueue {

    private final int MAX_CAPACITY = 10; // queue capacity
    private LinkedList
queue; // user queue

    public ExportQueue(LinkedList
queue) {
        this.queue = new LinkedList<>();
    }

    /** enqueue */
    public synchronized LinkedList
add(ExportUser sysUser) {
        while (queue.size() >= MAX_CAPACITY) {
            try {
                log.info("当前排队人已满，请等待");
                wait();
            } catch (InterruptedException e) {
                e.getMessage();
            }
        }
        queue.add(sysUser);
        log.info("目前导出队列排队人数：" + queue.size());
        notifyAll();
        return queue;
    }

    /** dequeue */
    public synchronized ExportUser getNextSysUser() {
        while (queue.isEmpty()) {
            try {
                wait();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        ExportUser sysUser = queue.remove();
        notifyAll(); // wake up
        return sysUser;
    }
}

The abstract export class leverages EasyExcel for million‑row Excel generation, handling pagination, response headers, and error logging.

package com.example.system.config;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.util.PageUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.example.system.api.domain.ExportUser;
import lombok.extern.slf4j.Slf4j;

import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.List;

@Slf4j
public abstract class AbstractExport
{

    public abstract void export(ExportUser sysUser) throws InterruptedException;

    public void export(HttpServletResponse response, int pageSize, T t, Class
k, String fileName) throws Exception {
        ExcelWriter writer = null;
        try {
            writer = getExcelWriter(response, fileName);
            int total = this.countExport(t);
            int loopCount = PageUtil.totalPage(total, pageSize);
            BeanUtil.setProperty(t, "pageSize", pageSize);
            for (int i = 0; i < loopCount; i++) {
                BeanUtil.setProperty(t, "pageNum", PageUtil.getStart(i + 1, pageSize));
                List
kList = this.getExportDetail(t);
                WriteSheet writeSheet = EasyExcel.writerSheet(fileName).head(k).build();
                writer.write(kList, writeSheet);
            }
        } catch (Exception e) {
            String msg = "导出" + fileName + "异常";
            log.error(msg, e);
            throw new Exception(msg + e);
        } finally {
            if (writer != null) {
                writer.finish();
            }
        }
    }

    public ExcelWriter getExcelWriter(HttpServletResponse response, String fileName) throws IOException {
        response.setContentType("application/vnd.ms-excel");
        response.setCharacterEncoding("utf-8");
        String fileNameUtf = URLEncoder.encode(fileName, "UTF-8").replaceAll("\\+", "%20");
        response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileNameUtf + ".xlsx");
        return EasyExcel.write(response.getOutputStream()).build();
    }

    public abstract void complexFillWithTable(T t, String fileName, HttpServletResponse response);
    public abstract int countExport(T t);
    public abstract List
getExportDetail(T t);
}

The concrete implementation ties the queue to the export process, logs actions, simulates processing delay, and removes the user from the queue after completion.

package com.example.system.service.impl;

import com.alibaba.excel.ExcelWriter;
import com.example.system.api.domain.ExportUser;
import com.example.system.config.AbstractExport;
import com.example.system.config.ExportQueue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;

@Service
@Slf4j
public class ExportImpl extends AbstractExport {

    @Autowired
    private ExportQueue exportQueue;

    @Override
    public void export(ExportUser sysUser) throws InterruptedException {
        log.info("导出文件方法执行～～～～～～～～～");
        LinkedList
queue = exportQueue.add(sysUser);
        log.info("导出队列：" + queue);
        Thread.sleep(20000); // simulate processing time
        ExportUser nextSysUser = exportQueue.getNextSysUser();
        log.info("移除后获取下一个排队的用户: " + nextSysUser.getUserName());
    }

    @Override
    public void export(HttpServletResponse response, int pageSize, Object o, Class k, String fileName) throws Exception {
        super.export(response, pageSize, o, k, fileName);
    }

    @Override
    public ExcelWriter getExcelWriter(HttpServletResponse response, String fileName) throws IOException {
        return super.getExcelWriter(response, fileName);
    }

    @Override
    public void complexFillWithTable(Object o, String fileName, HttpServletResponse response) {
        // not implemented
    }

    @Override
    public int countExport(Object o) {
        return 0;
    }

    @Override
    public List getExportDetail(Object o) {
        return null;
    }
}

A test controller launches export tasks in separate threads, demonstrating how concurrent requests are queued and processed.

package com.example.system.controller;

import com.example.system.api.domain.ExportUser;
import com.example.system.service.impl.ExportImpl;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping("/export")
@Slf4j
public class ExportController {

    @Autowired
    private ExportImpl export;

    @PostMapping("/exportFile")
    public void exportFile() {
        new Thread(new Runnable() {
            @SneakyThrows
            @Override
            public void run() {
                Thread thread1 = Thread.currentThread();
                ExportUser sysUser = new ExportUser();
                sysUser.setUserName(thread1.getName());
                export.export(sysUser);
            }
        }).start();
    }
}

Testing shows that when the queue reaches its maximum capacity of 10, additional export requests are blocked until space becomes available; subsequent requests are processed in order after earlier users finish.

The article notes that further work is needed for file metadata storage, OSS upload, download handling, and high‑concurrency considerations, and mentions that alternative queue solutions such as Redis could also be used.