FormData/Go分片/分块文件上传
2021-04-01 01:27
标签：ready count string png ceil ref erp cin color 如果你想构建一个简单的 更多解释MDN: https://developer.mozilla.org/zh-CN/docs/Web/API/FormData 分块(分片，统称分块了，确实只是发送一块数据)文件上传主要分2部分。 1. 前端js用file.slice可以从文件中切出一块一块的数据，然后用FormData包装一下，用XMLHttpRequest把切出来的数据块，一块一块send到server. 2. Server接收到的每一块都是一个multipart/form-data Form表单。可以在表单里放很多附属信息，文件名，大小，块大小，块索引，最总带上这块切出来的二进制数据。 multipart/form-data 数据 在Server存储文件，基本也就2种方案： 1. 直接创建一个对应大小的文件，按照每块数据的offset位置，写进去。 2. 每个传过来的数据块，保存成一个单独的数据块文件，最后把所有文件块合并成文件。 我这里只是做了一份简单的演示代码，基本上是不能用于生产环境的。 Index.html，直接把js写进去了 简单的Go server和保存文件，基本忽略所有的错误处理 目录截图，比较乱来 FormData/Go分片/分块文件上传 标签：ready count string png ceil ref erp cin color 原文地址：https://www.cnblogs.com/view85/p/12577335.htmlFormData 接口提供了一种表示表单数据的键值对的构造方式，经过它的数据可以使用 XMLHttpRequest.send() 方法送出，本接口和此方法都相当简单直接。如果送出时的编码类型被设为 "multipart/form-data"，它会使用和表单一样的格式。GET请求，并且通过的形式带有查询参数，可以将它直接传递给URLSearchParams。POST /upload HTTP/1.1
Host: localhost:8080
Content-Length: 2098072
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36
Content-Type: multipart/form-data; boundary=----WebKitFormBoundarymtng0xrR3ASR7wx7
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="file_name"
apache-maven-3.6.3-bin.zip
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="file_size"
9602303
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="block_size"
2097152
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="total_blocks"
5
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="break_error"
true
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="index"
3
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="data"; filename="blob"
Content-Type: application/octet-stream
(binary)
1 DOCTYPE html>
2 html>
3 head>
4 meta charset="utf8">
5 title>Multil-Blocks uploadtitle>
6 head>
7
8 body>
9 h2>Multil-Blocks uploadh2>
10
11 input id="file" type="file" />
12
13 input type="checkbox" id="multil_block_file">multil block fileinput>
14 button type="button" onclick="on_block_upload()">Block uploadbutton>
15 button type="button" onclick="on_concurrency_upload()">Concurrency uploadbutton>
16 hr/>
17
18 div>
19 label>File name: label>span id="file_name">span>
20 div>
21 div>
22 label>File size: label>span id="file_size">span>
23 div>
24 div>
25 label>Split blocks: label>span id="block_count">span>
26 div>
27
28 hr/>
29
30 p id="upload_info">p>
31
32 script>
33 var Block_Size = 1024 * 1024 * 2;
34
35 var el_file = document.getElementById(‘file‘);
36 var el_multil_block_file = document.getElementById(‘multil_block_file‘);
37 var el_file_name = document.getElementById(‘file_name‘);
38 var el_file_size = document.getElementById(‘file_size‘);
39 var el_block_count = document.getElementById(‘block_count‘);
40 var el_upload_info = document.getElementById(‘upload_info‘);
41
42 var file = null;
43 var total_blocks = 0;
44 var block_index = -1;
45 var block_index_random_arr = [];
46 var form_data = null;
47
48
49 el_file.onchange = function() {
50 if (this.files.length === 0) return;
51
52 file = this.files[0];
53 total_blocks = Math.ceil( file.size / Block_Size );
54
55 el_file_name.innerText = file.name;
56 el_file_size.innerText = file.size;
57 el_block_count.innerText = total_blocks;
58 }
59
60 function print_info(msg) {
61 el_upload_info.innerHTML += `${msg}br/>`;
62 }
63
64 function done() {
65 file = null;
66 total_blocks = 0;
67 block_index = -1;
68 form_data = null;
69
70 el_file.value = ‘‘;
71 }
72
73
74 function get_base_form_data() {
75 var base_data = new FormData();
76 base_data.append(‘file_name‘, file.name);
77 base_data.append(‘file_size‘, file.size);
78 base_data.append(‘block_size‘, Block_Size);
79 base_data.append(‘total_blocks‘, total_blocks);
80 base_data.append(‘break_error‘, true);
81 base_data.append(‘index‘, 0);
82 base_data.append(‘data‘, null);
83
84 return base_data
85 }
86
87
88 function build_block_index_random_arr() {
89 block_index_random_arr = new Array(total_blocks).fill(0).map((v,i) => i);
90 block_index_random_arr.sort((n, m) => Math.random() > .5 ? -1 : 1);
91
92 print_info(`Upload sequence: ${block_index_random_arr}`);
93 }
94
95
96 function post(index, success_cb, failed_cb) {
97 if (!form_data) {
98 form_data = get_base_form_data();
99 }
100 var start = index * Block_Size;
101 var end = Math.min(file.size, start + Block_Size);
102
103 form_data.set(‘index‘, index);
104 form_data.set(‘data‘, file.slice(start, end));
105
106 print_info(`Post ${index}/${total_blocks}, offset: ${start} -- ${end}`);
107
108
109 var xhr = new XMLHttpRequest();
110 xhr.open(‘POST‘, ‘/upload‘, true);
111 /*
112 Browser-based general content types
113 Content-Type: multipart/form-data; boundary=----WebKitFormBoundarysXH5DIES2XFMuLXL
114
115 Error content type:
116 xhr.setRequestHeader(‘Content-Type‘, ‘multipart/form-data‘);
117 Content-Type: multipart/form-data;
118 */
119 xhr.onreadystatechange = function() {
120
121 if (xhr.readyState === XMLHttpRequest.DONE) {
122
123 if (xhr.status >= 200 && xhr.status 300 && success_cb) {
124 return success_cb();
125 }
126
127 if (xhr.status >= 400 && failed_cb) {
128 failed_cb();
129 }
130 }
131 }
132
133 // xhr.onerror event
134 xhr.send(form_data);
135 }
136
137
138 function block_upload() {
139 if (!file) {
140 return;
141 }
142 if (block_index + 1 >= total_blocks) {
143 return done();
144 }
145
146 block_index++;
147 var index = block_index_random_arr[block_index];
148
149 post(index, block_upload);
150 }
151
152
153 function concurrency_upload() {
154 if (!file || total_blocks === 0) {
155 return;
156 }
157
158 build_block_index_random_arr();
159
160 form_data = get_base_form_data();
161 form_data.set(‘break_error‘, false);
162 form_data.set(‘multil_block‘, el_multil_block_file.checked);
163
164 for (var i of block_index_random_arr) {
165 ((idx) => {
166 post(idx, null, function() {
167 print_info(`Failed: ${idx}`);
168 setTimeout(() => post(idx), 1000);
169 });
170 })(i);
171 }
172 }
173
174
175 function on_block_upload() {
176 if (file) {
177 print_info(‘Block upload‘);
178
179 form_data = get_base_form_data();
180 form_data.set(‘multil_block‘, el_multil_block_file.checked);
181
182 build_block_index_random_arr();
183
184 block_index = -1;
185 block_upload();
186 }
187 }
188
189 function on_concurrency_upload() {
190 if (file) {
191 print_info(‘Concurrency upload‘);
192 concurrency_upload();
193 }
194 }
195 script>
196
197 body>
198 html>
1 package main
2
3 import (
4 "fmt"
5 "io/ioutil"
6 "log"
7 "net/http"
8 "os"
9 "path"
10 "path/filepath"
11 "regexp"
12 "strconv"
13 "strings"
14 "syscall"
15 "text/template"
16 )
17
18 type MultilBlockFile struct {
19 FileName string
20 Size int64
21 BlockSize int64
22 TotalBlocks int
23 Index int
24 Bufs []byte
25 BreakError bool
26 }
27
28 func fileIsExist(f string) bool {
29 _, err := os.Stat(f)
30 return err == nil || os.IsExist(err)
31 }
32
33 func lockFile(f *os.File) error {
34 err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
35 if err != nil {
36 return fmt.Errorf("get flock failed. err: %s", err)
37 }
38
39 return nil
40 }
41
42 func unlockFile(f *os.File) error {
43 defer f.Close()
44 return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
45 }
46
47 func singleFileSave(mbf *MultilBlockFile) error {
48
49 dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
50 filePath := path.Join(dir, "tmp", mbf.FileName)
51
52 offset := int64(mbf.Index) * mbf.BlockSize
53
54 fmt.Println(">>> Single file save ---------------------")
55 fmt.Printf("Save file: %s \n", filePath)
56 fmt.Printf("File offset: %d \n", offset)
57
58 var f *os.File
59 var needTruncate bool = false
60 if !fileIsExist(filePath) {
61 needTruncate = true
62 }
63
64 f, _ = os.OpenFile(filePath, syscall.O_CREAT|syscall.O_WRONLY, 0777)
65
66 err := lockFile(f)
67 if err != nil {
68 if mbf.BreakError {
69 log.Fatalf("get flock failed. err: %s", err)
70 } else {
71 return err
72 }
73 }
74
75 if needTruncate {
76 f.Truncate(mbf.Size)
77 }
78
79 f.WriteAt(mbf.Bufs, offset)
80
81 unlockFile(f)
82
83 return nil
84 }
85
86 func multilBlocksSave(mbf *MultilBlockFile) error {
87 dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
88 tmpFolderPath := path.Join(dir, "tmp")
89 tmpFileName := fmt.Sprintf("%s.%d", mbf.FileName, mbf.Index)
90 fileBlockPath := path.Join(tmpFolderPath, tmpFileName)
91
92 f, _ := os.OpenFile(fileBlockPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_TRUNC, 0777)
93 defer f.Close()
94
95 f.Write(mbf.Bufs)
96 f.Close()
97
98 re := regexp.MustCompile(`(?i:^` + mbf.FileName + `).\d$`)
99
100 files, _ := ioutil.ReadDir(tmpFolderPath)
101 matchFiles := make(map[string]bool)
102
103 for _, file := range files {
104 if file.IsDir() {
105 continue
106 }
107
108 fname := file.Name()
109 if re.MatchString(fname) {
110 matchFiles[fname] = true
111 }
112 }
113
114 if len(matchFiles) >= mbf.TotalBlocks {
115 lastFile, _ := os.OpenFile(path.Join(tmpFolderPath, mbf.FileName), syscall.O_CREAT|syscall.O_WRONLY, 0777)
116 lockFile(lastFile)
117
118 lastFile.Truncate(mbf.Size)
119
120 for name := range matchFiles {
121 tmpPath := path.Join(tmpFolderPath, name)
122
123 idxStr := name[strings.LastIndex(name, ".")+1:]
124 idx, _ := strconv.ParseInt(idxStr, 10, 32)
125
126 fmt.Printf("Match file: %s index: %d \n", name, idx)
127
128 data, _ := ioutil.ReadFile(tmpPath)
129
130 lastFile.WriteAt(data, idx*mbf.BlockSize)
131
132 os.Remove(tmpPath)
133 }
134 unlockFile(lastFile)
135 }
136
137 return nil
138 }
139
140 func indexHandle(w http.ResponseWriter, r *http.Request) {
141 tmp, _ := template.ParseFiles("./static/index.html")
142 tmp.Execute(w, "Index")
143 }
144
145 func uploadHandle(w http.ResponseWriter, r *http.Request) {
146
147 var mbf MultilBlockFile
148 mbf.FileName = r.FormValue("file_name")
149 mbf.Size, _ = strconv.ParseInt(r.FormValue("file_size"), 10, 64)
150 mbf.BlockSize, _ = strconv.ParseInt(r.FormValue("block_size"), 10, 64)
151 mbf.BreakError, _ = strconv.ParseBool(r.FormValue("break_error"))
152
153 var i int64
154 i, _ = strconv.ParseInt(r.FormValue("total_blocks"), 10, 32)
155 mbf.TotalBlocks = int(i)
156
157 i, _ = strconv.ParseInt(r.FormValue("index"), 10, 32)
158 mbf.Index = int(i)
159
160 d, _, _ := r.FormFile("data")
161 mbf.Bufs, _ = ioutil.ReadAll(d)
162
163 fmt.Printf(">>> Upload --------------------- \n")
164 fmt.Printf("File name: %s \n", mbf.FileName)
165 fmt.Printf("Size: %d \n", mbf.Size)
166 fmt.Printf("Block size: %d \n", mbf.BlockSize)
167 fmt.Printf("Total blocks: %d \n", mbf.TotalBlocks)
168 fmt.Printf("Index: %d \n", mbf.Index)
169 fmt.Println("Bufs len:", len(mbf.Bufs))
170
171 multilBlockFile, _ := strconv.ParseBool(r.FormValue("multil_block"))
172
173 var err error
174 if multilBlockFile {
175 err = multilBlocksSave(&mbf)
176 } else {
177 err = singleFileSave(&mbf)
178 }
179
180 if !mbf.BreakError && err != nil {
181 w.WriteHeader(400)
182 fmt.Fprintf(w, fmt.Sprintf("%s", err))
183 return
184 }
185
186 fmt.Fprintf(w, "ok")
187 }
188
189 func main() {
190 println("Listen on 8080")
191
192 http.HandleFunc("/", indexHandle)
193 http.HandleFunc("/upload", uploadHandle)
194
195 log.Fatal(http.ListenAndServe(":8080", nil))
196 }
