]>
Commit | Line | Data |
---|---|---|
c6060300 MT |
1 | To: vim_dev@googlegroups.com |
2 | Subject: Patch 7.3.427 | |
3 | Fcc: outbox | |
4 | From: Bram Moolenaar <Bram@moolenaar.net> | |
5 | Mime-Version: 1.0 | |
6 | Content-Type: text/plain; charset=UTF-8 | |
7 | Content-Transfer-Encoding: 8bit | |
8 | ------------ | |
9 | ||
10 | Patch 7.3.427 | |
11 | Problem: readfile() can be slow with long lines. | |
12 | Solution: Use realloc() instead of alloc(). (John Little) | |
13 | Files: src/eval.c | |
14 | ||
15 | ||
16 | *** ../vim-7.3.426/src/eval.c 2012-01-26 14:32:26.000000000 +0100 | |
17 | --- src/eval.c 2012-02-05 00:25:39.000000000 +0100 | |
18 | *************** | |
19 | *** 14325,14346 **** | |
20 | typval_T *rettv; | |
21 | { | |
22 | int binary = FALSE; | |
23 | char_u *fname; | |
24 | FILE *fd; | |
25 | ! listitem_T *li; | |
26 | ! #define FREAD_SIZE 200 /* optimized for text lines */ | |
27 | ! char_u buf[FREAD_SIZE]; | |
28 | ! int readlen; /* size of last fread() */ | |
29 | ! int buflen; /* nr of valid chars in buf[] */ | |
30 | ! int filtd; /* how much in buf[] was NUL -> '\n' filtered */ | |
31 | ! int tolist; /* first byte in buf[] still to be put in list */ | |
32 | ! int chop; /* how many CR to chop off */ | |
33 | ! char_u *prev = NULL; /* previously read bytes, if any */ | |
34 | ! int prevlen = 0; /* length of "prev" if not NULL */ | |
35 | ! char_u *s; | |
36 | ! int len; | |
37 | ! long maxline = MAXLNUM; | |
38 | ! long cnt = 0; | |
39 | ||
40 | if (argvars[1].v_type != VAR_UNKNOWN) | |
41 | { | |
42 | --- 14325,14343 ---- | |
43 | typval_T *rettv; | |
44 | { | |
45 | int binary = FALSE; | |
46 | + int failed = FALSE; | |
47 | char_u *fname; | |
48 | FILE *fd; | |
49 | ! char_u buf[(IOSIZE/256)*256]; /* rounded to avoid odd + 1 */ | |
50 | ! int io_size = sizeof(buf); | |
51 | ! int readlen; /* size of last fread() */ | |
52 | ! char_u *prev = NULL; /* previously read bytes, if any */ | |
53 | ! long prevlen = 0; /* length of data in prev */ | |
54 | ! long prevsize = 0; /* size of prev buffer */ | |
55 | ! long maxline = MAXLNUM; | |
56 | ! long cnt = 0; | |
57 | ! char_u *p; /* position in buf */ | |
58 | ! char_u *start; /* start of current line */ | |
59 | ||
60 | if (argvars[1].v_type != VAR_UNKNOWN) | |
61 | { | |
62 | *************** | |
63 | *** 14362,14410 **** | |
64 | return; | |
65 | } | |
66 | ||
67 | - filtd = 0; | |
68 | while (cnt < maxline || maxline < 0) | |
69 | { | |
70 | ! readlen = (int)fread(buf + filtd, 1, FREAD_SIZE - filtd, fd); | |
71 | ! buflen = filtd + readlen; | |
72 | ! tolist = 0; | |
73 | ! for ( ; filtd < buflen || readlen <= 0; ++filtd) | |
74 | ! { | |
75 | ! if (readlen <= 0 || buf[filtd] == '\n') | |
76 | ! { | |
77 | ! /* In binary mode add an empty list item when the last | |
78 | ! * non-empty line ends in a '\n'. */ | |
79 | ! if (!binary && readlen == 0 && filtd == 0 && prev == NULL) | |
80 | ! break; | |
81 | ||
82 | ! /* Found end-of-line or end-of-file: add a text line to the | |
83 | ! * list. */ | |
84 | ! chop = 0; | |
85 | ! if (!binary) | |
86 | ! while (filtd - chop - 1 >= tolist | |
87 | ! && buf[filtd - chop - 1] == '\r') | |
88 | ! ++chop; | |
89 | ! len = filtd - tolist - chop; | |
90 | ! if (prev == NULL) | |
91 | ! s = vim_strnsave(buf + tolist, len); | |
92 | else | |
93 | { | |
94 | ! s = alloc((unsigned)(prevlen + len + 1)); | |
95 | ! if (s != NULL) | |
96 | { | |
97 | ! mch_memmove(s, prev, prevlen); | |
98 | ! vim_free(prev); | |
99 | ! prev = NULL; | |
100 | ! mch_memmove(s + prevlen, buf + tolist, len); | |
101 | s[prevlen + len] = NUL; | |
102 | } | |
103 | } | |
104 | ! tolist = filtd + 1; | |
105 | ||
106 | ! li = listitem_alloc(); | |
107 | ! if (li == NULL) | |
108 | { | |
109 | vim_free(s); | |
110 | break; | |
111 | } | |
112 | li->li_tv.v_type = VAR_STRING; | |
113 | --- 14359,14419 ---- | |
114 | return; | |
115 | } | |
116 | ||
117 | while (cnt < maxline || maxline < 0) | |
118 | { | |
119 | ! readlen = (int)fread(buf, 1, io_size, fd); | |
120 | ! | |
121 | ! /* This for loop processes what was read, but is also entered at end | |
122 | ! * of file so that either: | |
123 | ! * - an incomplete line gets written | |
124 | ! * - a "binary" file gets an empty line at the end if it ends in a | |
125 | ! * newline. */ | |
126 | ! for (p = buf, start = buf; | |
127 | ! p < buf + readlen || (readlen <= 0 && (prevlen > 0 || binary)); | |
128 | ! ++p) | |
129 | ! { | |
130 | ! if (*p == '\n' || readlen <= 0) | |
131 | ! { | |
132 | ! listitem_T *li; | |
133 | ! char_u *s = NULL; | |
134 | ! long_u len = p - start; | |
135 | ||
136 | ! /* Finished a line. Remove CRs before NL. */ | |
137 | ! if (readlen > 0 && !binary) | |
138 | ! { | |
139 | ! while (len > 0 && start[len - 1] == '\r') | |
140 | ! --len; | |
141 | ! /* removal may cross back to the "prev" string */ | |
142 | ! if (len == 0) | |
143 | ! while (prevlen > 0 && prev[prevlen - 1] == '\r') | |
144 | ! --prevlen; | |
145 | ! } | |
146 | ! if (prevlen == 0) | |
147 | ! s = vim_strnsave(start, len); | |
148 | else | |
149 | { | |
150 | ! /* Change "prev" buffer to be the right size. This way | |
151 | ! * the bytes are only copied once, and very long lines are | |
152 | ! * allocated only once. */ | |
153 | ! if ((s = vim_realloc(prev, prevlen + len + 1)) != NULL) | |
154 | { | |
155 | ! mch_memmove(s + prevlen, start, len); | |
156 | s[prevlen + len] = NUL; | |
157 | + prev = NULL; /* the list will own the string */ | |
158 | + prevlen = prevsize = 0; | |
159 | } | |
160 | } | |
161 | ! if (s == NULL) | |
162 | ! { | |
163 | ! do_outofmem_msg((long_u) prevlen + len + 1); | |
164 | ! failed = TRUE; | |
165 | ! break; | |
166 | ! } | |
167 | ||
168 | ! if ((li = listitem_alloc()) == NULL) | |
169 | { | |
170 | vim_free(s); | |
171 | + failed = TRUE; | |
172 | break; | |
173 | } | |
174 | li->li_tv.v_type = VAR_STRING; | |
175 | *************** | |
176 | *** 14412,14485 **** | |
177 | li->li_tv.vval.v_string = s; | |
178 | list_append(rettv->vval.v_list, li); | |
179 | ||
180 | ! if (++cnt >= maxline && maxline >= 0) | |
181 | ! break; | |
182 | ! if (readlen <= 0) | |
183 | break; | |
184 | } | |
185 | ! else if (buf[filtd] == NUL) | |
186 | ! buf[filtd] = '\n'; | |
187 | #ifdef FEAT_MBYTE | |
188 | ! else if (buf[filtd] == 0xef | |
189 | ! && enc_utf8 | |
190 | ! && filtd + 2 < buflen | |
191 | ! && !binary | |
192 | ! && buf[filtd + 1] == 0xbb | |
193 | ! && buf[filtd + 2] == 0xbf) | |
194 | ! { | |
195 | ! /* remove utf-8 byte order mark */ | |
196 | ! mch_memmove(buf + filtd, buf + filtd + 3, buflen - filtd - 3); | |
197 | ! --filtd; | |
198 | ! buflen -= 3; | |
199 | } | |
200 | #endif | |
201 | ! } | |
202 | ! if (readlen <= 0) | |
203 | ! break; | |
204 | ||
205 | ! if (tolist == 0) | |
206 | { | |
207 | ! if (buflen >= FREAD_SIZE / 2) | |
208 | { | |
209 | ! /* "buf" is full, need to move text to an allocated buffer */ | |
210 | ! if (prev == NULL) | |
211 | { | |
212 | ! prev = vim_strnsave(buf, buflen); | |
213 | ! prevlen = buflen; | |
214 | } | |
215 | ! else | |
216 | { | |
217 | ! s = alloc((unsigned)(prevlen + buflen)); | |
218 | ! if (s != NULL) | |
219 | ! { | |
220 | ! mch_memmove(s, prev, prevlen); | |
221 | ! mch_memmove(s + prevlen, buf, buflen); | |
222 | ! vim_free(prev); | |
223 | ! prev = s; | |
224 | ! prevlen += buflen; | |
225 | ! } | |
226 | } | |
227 | ! filtd = 0; | |
228 | } | |
229 | } | |
230 | ! else | |
231 | ! { | |
232 | ! mch_memmove(buf, buf + tolist, buflen - tolist); | |
233 | ! filtd -= tolist; | |
234 | ! } | |
235 | ! } | |
236 | ||
237 | /* | |
238 | * For a negative line count use only the lines at the end of the file, | |
239 | * free the rest. | |
240 | */ | |
241 | ! if (maxline < 0) | |
242 | while (cnt > -maxline) | |
243 | { | |
244 | listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first); | |
245 | --cnt; | |
246 | } | |
247 | ||
248 | vim_free(prev); | |
249 | fclose(fd); | |
250 | } | |
251 | --- 14421,14529 ---- | |
252 | li->li_tv.vval.v_string = s; | |
253 | list_append(rettv->vval.v_list, li); | |
254 | ||
255 | ! start = p + 1; /* step over newline */ | |
256 | ! if ((++cnt >= maxline && maxline >= 0) || readlen <= 0) | |
257 | break; | |
258 | } | |
259 | ! else if (*p == NUL) | |
260 | ! *p = '\n'; | |
261 | #ifdef FEAT_MBYTE | |
262 | ! /* Check for utf8 "bom"; U+FEFF is encoded as EF BB BF. Do this | |
263 | ! * when finding the BF and check the previous two bytes. */ | |
264 | ! else if (*p == 0xbf && enc_utf8 && !binary) | |
265 | ! { | |
266 | ! /* Find the two bytes before the 0xbf. If p is at buf, or buf | |
267 | ! * + 1, these may be in the "prev" string. */ | |
268 | ! char_u back1 = p >= buf + 1 ? p[-1] | |
269 | ! : prevlen >= 1 ? prev[prevlen - 1] : NUL; | |
270 | ! char_u back2 = p >= buf + 2 ? p[-2] | |
271 | ! : p == buf + 1 && prevlen >= 1 ? prev[prevlen - 1] | |
272 | ! : prevlen >= 2 ? prev[prevlen - 2] : NUL; | |
273 | ! | |
274 | ! if (back2 == 0xef && back1 == 0xbb) | |
275 | ! { | |
276 | ! char_u *dest = p - 2; | |
277 | ! | |
278 | ! /* Usually a BOM is at the beginning of a file, and so at | |
279 | ! * the beginning of a line; then we can just step over it. | |
280 | ! */ | |
281 | ! if (start == dest) | |
282 | ! start = p + 1; | |
283 | ! else | |
284 | ! { | |
285 | ! /* have to shuffle buf to close gap */ | |
286 | ! int adjust_prevlen = 0; | |
287 | ! | |
288 | ! if (dest < buf) | |
289 | ! { | |
290 | ! adjust_prevlen = buf - dest; /* must be 1 or 2 */ | |
291 | ! dest = buf; | |
292 | ! } | |
293 | ! if (readlen > p - buf + 1) | |
294 | ! mch_memmove(dest, p + 1, readlen - (p - buf) - 1); | |
295 | ! readlen -= 3 - adjust_prevlen; | |
296 | ! prevlen -= adjust_prevlen; | |
297 | ! p = dest - 1; | |
298 | ! } | |
299 | ! } | |
300 | } | |
301 | #endif | |
302 | ! } /* for */ | |
303 | ||
304 | ! if (failed || (cnt >= maxline && maxline >= 0) || readlen <= 0) | |
305 | ! break; | |
306 | ! if (start < p) | |
307 | { | |
308 | ! /* There's part of a line in buf, store it in "prev". */ | |
309 | ! if (p - start + prevlen >= prevsize) | |
310 | { | |
311 | ! /* need bigger "prev" buffer */ | |
312 | ! char_u *newprev; | |
313 | ! | |
314 | ! /* A common use case is ordinary text files and "prev" gets a | |
315 | ! * fragment of a line, so the first allocation is made | |
316 | ! * small, to avoid repeatedly 'allocing' large and | |
317 | ! * 'reallocing' small. */ | |
318 | ! if (prevsize == 0) | |
319 | ! prevsize = p - start; | |
320 | ! else | |
321 | { | |
322 | ! long grow50pc = (prevsize * 3) / 2; | |
323 | ! long growmin = (p - start) * 2 + prevlen; | |
324 | ! prevsize = grow50pc > growmin ? grow50pc : growmin; | |
325 | } | |
326 | ! if ((newprev = vim_realloc(prev, prevsize)) == NULL) | |
327 | { | |
328 | ! do_outofmem_msg((long_u)prevsize); | |
329 | ! failed = TRUE; | |
330 | ! break; | |
331 | } | |
332 | ! prev = newprev; | |
333 | } | |
334 | + /* Add the line part to end of "prev". */ | |
335 | + mch_memmove(prev + prevlen, start, p - start); | |
336 | + prevlen += p - start; | |
337 | } | |
338 | ! } /* while */ | |
339 | ||
340 | /* | |
341 | * For a negative line count use only the lines at the end of the file, | |
342 | * free the rest. | |
343 | */ | |
344 | ! if (!failed && maxline < 0) | |
345 | while (cnt > -maxline) | |
346 | { | |
347 | listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first); | |
348 | --cnt; | |
349 | } | |
350 | ||
351 | + if (failed) | |
352 | + { | |
353 | + list_free(rettv->vval.v_list, TRUE); | |
354 | + /* readfile doc says an empty list is returned on error */ | |
355 | + rettv->vval.v_list = list_alloc(); | |
356 | + } | |
357 | + | |
358 | vim_free(prev); | |
359 | fclose(fd); | |
360 | } | |
361 | *** ../vim-7.3.426/src/version.c 2012-02-04 23:34:57.000000000 +0100 | |
362 | --- src/version.c 2012-02-05 00:38:34.000000000 +0100 | |
363 | *************** | |
364 | *** 716,717 **** | |
365 | --- 716,719 ---- | |
366 | { /* Add new patch number below this line */ | |
367 | + /**/ | |
368 | + 427, | |
369 | /**/ | |
370 | ||
371 | -- | |
372 | One difference between a man and a machine is that a machine is quiet | |
373 | when well oiled. | |
374 | ||
375 | /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ | |
376 | /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ | |
377 | \\\ an exciting new programming language -- http://www.Zimbu.org /// | |
378 | \\\ help me help AIDS victims -- http://ICCF-Holland.org /// |