在日常生活中,文件上传相关的操作随处可见,大到处理大数据量的文件,小到头像上传,都离不开文件上传操作,但是当一个文件的大小超过了某个阈值时,这个文件的上传过程就会变得及其的慢,且会消耗大量网络资源,这是我们不愿意看到的,所以,文件分片上传孕育而生。
什么是文件分片上传?
文件分片上传就是将一整个文件分为几个小块,然后将这几个小块分别传送给服务器,从而实现分片上传。
上图为文件分片的图解,在本图中,我们假定每一个分片都为67MB。(只是演示,实际文件分片需要考虑更多细节)
如果当我们分片到最后一片的时候,我们就会直接将剩余所有空间存放到一个切片中,不管大小是否足够我们指定的大小。
注意:这里的最后一片是指剩余的文件大小小于等于我们分片指定大小的情况。
文件分片时需要考虑什么?
在进行文件分片时,我们需要按照实际情况下文件大小来指定每一个切片的大小。并且需要在切片后将所有切片数量做记录,具体流程将以列表形式呈现:
前端
获取文件,并规定一些常量(如切片大小,和后端约定的状态信息等等)开始文件切片,并将切片存储到数组中将切片数组中的切片转换为二进制形式(原数组不变,只取数据)并添加到缓冲区(SparkMD5库提供的缓冲区)中确保所有切片全都存入缓冲区(这时候缓冲区内的其实就是我们的整体文件,所有切片都合并了),然后计算文件hash.开始对后端进行数据交互(上传分片,提示合并,检查是否已经上传文件 等)后端
从前端获取相关信息(如文件hash,文件名,切片文件等)检查是否已经上传过相同文件等待所有切片文件存储完成,并接收前端的合并通知(这一条看个人,也可以在后端直接计算是否拿到所有切片)确保拿到所有切片文件后,开始读取切片文件的二进制信息,并将其添加到缓冲区中读取完全部文件后,将缓冲区数据写入指定文件中将切片文件全部删除以上是文件分片上传时前后端的基础流程(可能有些地方写的不够严谨,希望各位大佬指教)
特别注意:在文件合并时要注意分片文件合并的顺序问题,如果顺序颠倒,那文件自然无法正常显示。
个人建议所有分片文件命名后面跟上一个索引.
代码实战
声明:此代码没有考虑过多细节,只是作为一个基础展示的案例。
前端
<!doctype html><html lang="en"><head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0"> <meta http-equiv="X-UA-Compatible" content="ie=edge"> <title>Document</title> <style> .msg{ font-size: 20px; font-weight: bold; } </style></head><body><input type="file"><p class="msg"></p><script src="js/axios.js"></script><script src="js/spark-md5.js"></script><script> const statusCode = { UPLOAD_SUCCESS: 200, NOT_UPLOAD: 202, ALREADY_UPLOAD: 1000, UPLOAD_FAILED: 1004 } let chunkSize = 2 * 1024 * 1024 let msg = document.querySelector(".msg") let file = document.querySelector("input[type='file']") file.addEventListener("change", async (e) => { let fileList = e.target.files let file = fileList[0] let chunkArr = chunk(file, chunkSize) let fileHash = await hash(chunkArr) let filename = file.name //false:没上传 true:上传过了 let hasUpload = await check(fileHash, filename) if (!hasUpload) { let promises = [] for (let i = 0; i < chunkArr.length; i++) { //将最后的返回结果添加到数组中 let res = await upload(fileHash, chunkArr, i, filename) promises.push(res) } Promise.all(promises).then(res => { mergeNotify(fileHash, filename, chunkArr.length) msg.innerHTML="文件上传成功" msg.style.color="green" }).catch(err => { console.error(err) }) } else { //文件上传过了,无需再次上传 msg.innerHTML="文件已经上传!!" msg.style.color="red" } }) /** * * @param file 文件File对象 * @param chunkSize 每一个切片的大小 * @return {[]} 返回切片数组 */ const chunk = (file, chunkSize) => { let res = [] for (let i = 0; i < file.size; i += chunkSize) { res.push(file.slice(i, i + chunkSize)) } return res } /** * * @param chunks 切片数组 * @return string 返回文件hash */ const hash = async (chunks) => { let sparkMD5 = new SparkMD5.ArrayBuffer() //存储每个切片加密的任务状态,全部完成后,才会返回最终hash let promises = [] //将切片数组所有切片转为二进制,并将其合并为一个完整文件 for (let i = 0; i < chunks.length; i++) { //由于hash加密耗时,所以我们采用异步 let promise = new Promise((resolve, reject) => { let fileReader = new FileReader()//使用fileReader对象将文件切片转为二进制 fileReader.readAsArrayBuffer(chunks[i]) fileReader.onload = (e) => { //添加到SparkMD5中,等所有切片添加完毕后,获取最终哈希 sparkMD5.append(e.target.result) //每次添加成功后返回一个成功状态 resolve() } fileReader.onerror = (e) => { reject(e.target.error) } }) //将该promise任务添加到promise数组中 promises.push(promise) } //当所有加密任务全都完成后,返回加密后的完整文件hash return await Promise.all(promises).then(res => { return sparkMD5.end() }).catch(err => { console.error("Hash加密出现问题") }) } /*** * * @param hash 文件hash * @param chunks 切片数组 * @param currentIndex 当前切片索引 * @param filename 文件名 * @return 返回Promise,用于检测当前切片是否上传成功 */ const upload = (hash, chunks, currentIndex, filename) => { return new Promise((resolve, reject) => { let formData = new FormData() formData.append("hash", hash) formData.append("chunkIndex", currentIndex) formData.append("filename", filename) formData.append("chunkBody", chunks[currentIndex]) axios.post("http://localhost:8080/upload", formData).then(res => { //出现无法判断是否成功的问题,推荐判断是否成功在Promise.all中判断 resolve("") }).catch(err => { reject(err) }) }) } /*** * 通知后端接口:可以开始合并任务了 * @param hash 文件hash * @param filename 文件名 */ const mergeNotify = (hash, filename, chunksLen) => { let formData = new FormData() formData.append("filename", filename) formData.append("fileHash", hash) formData.append("totalChunk", chunksLen) axios.post("http://localhost:8080/merge", formData).then(res => {}) } /** * 检查文件是否上传 * @param hash 文件hash * @param filename 文件名 * @return {Promise<Boolean>} 返回一个Promise对象 */ const check = async (hash, filename) => { let formData = new FormData() formData.append("filename", filename) formData.append("fileHash", hash) let hasUpload = axios.post("http://localhost:8080/check", formData).then(res => { let result; //判断是否上传过该文件 if (res.data.code === statusCode.NOT_UPLOAD) { result = false } else { result = true } //返回promise对象 return Promise.resolve(result) }) return hasUpload }</script></body></html>后端
entity
BaseFile
package com.cc.fileupload.entity;/** * @author CC * @date Created in 2024/2/7 12:15 */public class BaseFile { /** * 文件hash */ private String fileHash; public BaseFile() { } public BaseFile(String fileHash, String filename) { this.fileHash = fileHash; this.filename = filename; } /** * 文件名 */ private String filename; @Override public String toString() { return "BaseFile{" + "fileHash='" + fileHash + '\'' + ", filename='" + filename + '\'' + '}'; } public String getFileHash() { return fileHash; } public void setFileHash(String fileHash) { this.fileHash = fileHash; } public String getFilename() { return filename; } public void setFilename(String filename) { this.filename = filename; }}MergeFile
package com.cc.fileupload.entity;/** * @author CC * @date Created in 2024/2/7 11:27 */public class MergeFile { /** * 文件名 */ private String filename; /** * 文件hash */ private String fileHash; /** * 切片总数 */ private Integer totalChunk; public String getFilename() { return filename; } public void setFilename(String filename) { this.filename = filename; } public String getFileHash() { return fileHash; } public void setFileHash(String fileHash) { this.fileHash = fileHash; } public Integer getTotalChunk() { return totalChunk; } @Override public String toString() { return "MergeFile{" + "filename='" + filename + '\'' + ", fileHash='" + fileHash + '\'' + ", totalChunk=" + totalChunk + '}'; } public void setTotalChunk(Integer totalChunk) { this.totalChunk = totalChunk; } public MergeFile() { } public MergeFile(String filename, String fileHash, Integer totalChunk) { this.filename = filename; this.fileHash = fileHash; this.totalChunk = totalChunk; }}UploadFile
package com.cc.fileupload.entity;import org.springframework.web.bind.annotation.RequestParam;import org.springframework.web.multipart.MultipartFile;/** * @author CC * @date Created in 2024/2/7 10:33 */public class UploadFile { /** * 传入的切片文件 */ private MultipartFile chunkBody; /** * 文件hash */ private String hash; /** * 文件名 */ private String filename; /** * 当前切片的索引号 */ private Integer chunkIndex; public MultipartFile getChunkBody() { return chunkBody; } public void setChunkBody(MultipartFile chunkBody) { this.chunkBody = chunkBody; } public String getHash() { return hash; } public void setHash(String hash) { this.hash = hash; } public String getFilename() { return filename; } public void setFilename(String filename) { this.filename = filename; } public Integer getChunkIndex() { return chunkIndex; } public void setChunkIndex(Integer chunkIndex) { this.chunkIndex = chunkIndex; } @Override public String toString() { return "UploadFile{" + "chunkBody=" + chunkBody + ", hash='" + hash + '\'' + ", filename='" + filename + '\'' + ", chunkIndex=" + chunkIndex + '}'; }}util
Helper
package com.cc.fileupload.util;/** * @author CC * @date Created in 2024/2/7 10:49 */public class Helper { /** * 构建切片文件名 * * @param baseName 基础文件名 * @param index 文件索引 * @return 返回切片文件名 */ public static String buildChunkName(String baseName, Integer index) { int i = baseName.lastIndexOf("."); String prefix = baseName.substring(0, i).replaceAll("\\.", "_"); return prefix + "_part_" + index; } public static <T> ResultFormat<T> getReturnMsg(Integer code, T data, String msg) { return new ResultFormat<T>(data, msg, code); } public static <T> ResultFormat<T> getReturnMsg(Integer code, T data) { return new ResultFormat<T>(data, code); } public static ResultFormat<String> getReturnMsg(Integer code, String msg) { return new ResultFormat<>(msg, code); } public static ResultFormat<Integer> getReturnMsg(Integer code){ return new ResultFormat<>(code); }//// public static void main(String[] args) {// String s = buildChunkName("test.xx.txt", 1);// System.out.println(s);// }}ResultFormat
package com.cc.fileupload.util;/** * @author CC * @date Created in 2024/2/7 11:46 */public class ResultFormat<T> { private T data; private String msg; private Integer code; @Override public String toString() { return "{" + "data=" + data + ", msg='" + msg + '\'' + ", code=" + code + '}'; } public T getData() { return data; } public void setData(T data) { this.data = data; } public String getMsg() { return msg; } public void setMsg(String msg) { this.msg = msg; } public Integer getCode() { return code; } public void setCode(Integer code) { this.code = code; } public ResultFormat(String msg, Integer code) { this.msg = msg; this.code = code; } public ResultFormat(Integer code) { this.code = code; } public ResultFormat(T data, Integer code) { this.data = data; this.code = code; } public ResultFormat(T data, String msg, Integer code) { this.data = data; this.msg = msg; this.code = code; }}StatusCode
package com.cc.fileupload.util;/** * @author CC * @date Created in 2024/2/7 11:46 */public enum StatusCode { UPLOAD_SUCCESS(200), NOT_UPLOAD(202), ALREADY_UPLOAD(1000), UPLOAD_FAILED(1004); private java.lang.Integer code; StatusCode(java.lang.Integer code) { this.code = code; } public java.lang.Integer getCode() { return code; } public void setCode(java.lang.Integer code) { this.code = code; }}service
UploadService
package com.cc.fileupload.service;import com.cc.fileupload.entity.BaseFile;import com.cc.fileupload.entity.MergeFile;import com.cc.fileupload.entity.UploadFile;import com.cc.fileupload.util.ResultFormat;import java.io.File;/** * @author CC * @date Created in 2024/2/7 10:46 */public interface UploadService { /** * 上传文件并保存切片的操作 * * @param uploadFile 文件上传实体类 * @return 返回状态信息 */ ResultFormat upload(UploadFile uploadFile); /** * 合并文件切片 * * @param mergeFile 合并文件实体类 */ void merge(MergeFile mergeFile); /** * 对文件的切片做删除操作 * @param mergeFile 合并文件实体类 */ void deleteChunks(MergeFile mergeFile); /** * * @param baseFile 检查文件是否已经上传 * @return 返回状态信息 */ ResultFormat<Integer> checkHasUpload(BaseFile baseFile);}IUploadService
package com.cc.fileupload.service.impl;import com.cc.fileupload.entity.BaseFile;import com.cc.fileupload.entity.MergeFile;import com.cc.fileupload.entity.UploadFile;import com.cc.fileupload.service.UploadService;import com.cc.fileupload.util.Helper;import com.cc.fileupload.util.ResultFormat;import com.cc.fileupload.util.StatusCode;import org.springframework.stereotype.Service;import org.springframework.web.multipart.MultipartFile;import java.io.File;import java.io.FileOutputStream;import java.io.IOException;import java.io.OutputStream;import java.nio.file.Files;/** * @author CC * @date Created in 2024/2/7 10:46 */@Servicepublic class IUploadService implements UploadService { private static final String BASE_PATH = "D:\\桌面\\图片"; @Override public ResultFormat<java.lang.Integer> checkHasUpload(BaseFile mergeFile) { String fileHash = mergeFile.getFileHash(); String filename = mergeFile.getFilename(); File folder = new File(BASE_PATH, fileHash); if (folder.exists()) { File file = new File(folder, filename); if (file.exists()) { return Helper.getReturnMsg(StatusCode.ALREADY_UPLOAD.getCode()); } } return Helper.getReturnMsg(StatusCode.NOT_UPLOAD.getCode()); } @Override public ResultFormat upload(UploadFile uploadFile) { String filename = uploadFile.getFilename(); String hash = uploadFile.getHash(); java.lang.Integer currentChunkIndex = uploadFile.getChunkIndex(); MultipartFile chunkBody = uploadFile.getChunkBody(); //根据hash来创建文件夹,有助于检测是否上传 File folder = new File(BASE_PATH, hash); if (!folder.exists()) { folder.mkdirs(); } //这里获取需要写入的文件路径和文件名 File file1 = new File(folder, Helper.buildChunkName(filename, currentChunkIndex)); try { //文件写入 chunkBody.transferTo(file1); return Helper.getReturnMsg(StatusCode.UPLOAD_SUCCESS.getCode(), "上传成功"); } catch (IOException e) { System.out.println("出现错误"); e.printStackTrace(); } //对文件进行写入 return Helper.getReturnMsg(StatusCode.UPLOAD_FAILED.getCode(), "上传失败"); } @Override public void deleteChunks(MergeFile mergeFile) { File hashFolder = new File(BASE_PATH, mergeFile.getFileHash()); java.lang.Integer totalChunk = mergeFile.getTotalChunk(); String filename = mergeFile.getFilename(); for (int i = 0; i < totalChunk; i++) { //获取切片 File tmpChunkFile = new File(hashFolder, Helper.buildChunkName(filename, i)); tmpChunkFile.delete(); } } @Override public void merge(MergeFile mergeFile) { String hash = mergeFile.getFileHash(); String filename = mergeFile.getFilename(); java.lang.Integer totalChunk = mergeFile.getTotalChunk(); //文件hash的Folder File hashFolder = new File(BASE_PATH, hash); OutputStream os = null; //检查是否有该hash目录 try { if (hashFolder.exists()) { //指定最后输出的文件名 os = new FileOutputStream(new File(hashFolder, filename)); for (int i = 0; i < totalChunk; i++) { //获取切片 File tmpChunkFile = new File(hashFolder, Helper.buildChunkName(filename, i)); //数据读取并写入缓存区 byte[] bytes = Files.readAllBytes(tmpChunkFile.toPath()); //将每一个切片数据读取写入缓存区 os.write(bytes); } //在将每一个切片的字节全都写入缓冲区后,最后合并输出文件 os.flush(); //输出后清理临时文件 deleteChunks(mergeFile); } } catch (IOException e) { e.printStackTrace(); } finally { //资源关闭 if (os != null) { try { os.close(); } catch (IOException e) { e.printStackTrace(); } } } }}controller
UploadController
package com.cc.fileupload.controller;import com.cc.fileupload.entity.BaseFile;import com.cc.fileupload.entity.MergeFile;import com.cc.fileupload.entity.UploadFile;import com.cc.fileupload.service.UploadService;import com.cc.fileupload.util.ResultFormat;import org.springframework.web.bind.annotation.*;import javax.annotation.Resource;/** * @author CC * @date Created in 2024/2/7 9:46 */@RestController@CrossOriginpublic class UploadController { @Resource private UploadService uploadService; @RequestMapping("/upload") public ResultFormat upload(@ModelAttribute UploadFile uploadFile) { System.out.println("上传"); return uploadService.upload(uploadFile); } @RequestMapping("/merge") public void merge(@ModelAttribute MergeFile mergeFile) { uploadService.merge(mergeFile); } @RequestMapping("/check") public ResultFormat check(@ModelAttribute BaseFile file) { System.out.println("检查"); return uploadService.checkHasUpload(file); }}github链接
前端:GitHub - wewCc/fileUpload_frontend: 文件上传前端文件上传前端. Contribute to wewCc/fileUpload_frontend development by creating an account on GitHub.
https://github.com/wewCc/fileUpload_frontend
后端:https://github.com/wewCc/fileUpload
https://github.com/wewCc/fileUpload