]>
Commit | Line | Data |
---|---|---|
1d13e637 AF |
1 | From 252499c1513c45764d039af8732cd97b37c8c494 Mon Sep 17 00:00:00 2001 |
2 | From: Volker Lendecke <vl@samba.org> | |
3 | Date: Thu, 9 Feb 2017 15:40:39 +0100 | |
4 | Subject: [PATCH 1/3] smbd: Streamline get_ea_names_from_file | |
5 | ||
6 | Signed-off-by: Volker Lendecke <vl@samba.org> | |
7 | Reviewed-by: Ralph Boehme <slow@samba.org> | |
8 | Backported-by: Andreas Schneider <asn@samba.org> | |
9 | Backported-from: 27daed8fcf95eed2df112dc1c30c3a40b5c9565b | |
10 | --- | |
11 | source3/smbd/trans2.c | 89 +++++++++++++++++++++++++++++---------------------- | |
12 | 1 file changed, 51 insertions(+), 38 deletions(-) | |
13 | ||
14 | diff --git a/source3/smbd/trans2.c b/source3/smbd/trans2.c | |
15 | index 98fd2af..49cfe9f 100644 | |
16 | --- a/source3/smbd/trans2.c | |
17 | +++ b/source3/smbd/trans2.c | |
18 | @@ -201,12 +201,14 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn, | |
19 | files_struct *fsp, const char *fname, | |
20 | char ***pnames, size_t *pnum_names) | |
21 | { | |
22 | + char smallbuf[1024]; | |
23 | /* Get a list of all xattrs. Max namesize is 64k. */ | |
24 | size_t ea_namelist_size = 1024; | |
25 | - char *ea_namelist = NULL; | |
26 | + char *ea_namelist = smallbuf; | |
27 | + char *to_free = NULL; | |
28 | ||
29 | char *p; | |
30 | - char **names, **tmp; | |
31 | + char **names; | |
32 | size_t num_names; | |
33 | ssize_t sizeret = -1; | |
34 | NTSTATUS status; | |
35 | @@ -228,25 +230,24 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn, | |
36 | return NT_STATUS_OK; | |
37 | } | |
38 | ||
39 | - /* | |
40 | - * TALLOC the result early to get the talloc hierarchy right. | |
41 | - */ | |
42 | - | |
43 | - names = TALLOC_ARRAY(mem_ctx, char *, 1); | |
44 | - if (names == NULL) { | |
45 | - DEBUG(0, ("talloc failed\n")); | |
46 | - return NT_STATUS_NO_MEMORY; | |
47 | + if (fsp && fsp->fh->fd != -1) { | |
48 | + sizeret = SMB_VFS_FLISTXATTR(fsp, ea_namelist, | |
49 | + ea_namelist_size); | |
50 | + } else { | |
51 | + sizeret = SMB_VFS_LISTXATTR(conn, | |
52 | + fname, | |
53 | + ea_namelist, | |
54 | + ea_namelist_size); | |
55 | } | |
56 | ||
57 | - while (ea_namelist_size <= 65536) { | |
58 | - | |
59 | - ea_namelist = TALLOC_REALLOC_ARRAY( | |
60 | - names, ea_namelist, char, ea_namelist_size); | |
61 | + if ((sizeret == -1) && (errno == ERANGE)) { | |
62 | + ea_namelist_size = 65536; | |
63 | + ea_namelist = TALLOC_ARRAY(mem_ctx, char, ea_namelist_size); | |
64 | if (ea_namelist == NULL) { | |
65 | DEBUG(0, ("talloc failed\n")); | |
66 | - TALLOC_FREE(names); | |
67 | return NT_STATUS_NO_MEMORY; | |
68 | } | |
69 | + to_free = ea_namelist; | |
70 | ||
71 | if (fsp && fsp->fh->fd != -1) { | |
72 | sizeret = SMB_VFS_FLISTXATTR(fsp, ea_namelist, | |
73 | @@ -255,25 +256,18 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn, | |
74 | sizeret = SMB_VFS_LISTXATTR(conn, fname, ea_namelist, | |
75 | ea_namelist_size); | |
76 | } | |
77 | - | |
78 | - if ((sizeret == -1) && (errno == ERANGE)) { | |
79 | - ea_namelist_size *= 2; | |
80 | - } | |
81 | - else { | |
82 | - break; | |
83 | - } | |
84 | } | |
85 | ||
86 | if (sizeret == -1) { | |
87 | - TALLOC_FREE(names); | |
88 | - return map_nt_error_from_unix(errno); | |
89 | + status = map_nt_error_from_unix(errno); | |
90 | + TALLOC_FREE(to_free); | |
91 | + return status; | |
92 | } | |
93 | ||
94 | - DEBUG(10, ("get_ea_list_from_file: ea_namelist size = %u\n", | |
95 | - (unsigned int)sizeret)); | |
96 | + DEBUG(10, ("ea_namelist size = %zd\n", sizeret)); | |
97 | ||
98 | if (sizeret == 0) { | |
99 | - TALLOC_FREE(names); | |
100 | + TALLOC_FREE(to_free); | |
101 | return NT_STATUS_OK; | |
102 | } | |
103 | ||
104 | @@ -282,7 +276,7 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn, | |
105 | */ | |
106 | ||
107 | if (ea_namelist[sizeret-1] != '\0') { | |
108 | - TALLOC_FREE(names); | |
109 | + TALLOC_FREE(to_free); | |
110 | return NT_STATUS_INTERNAL_ERROR; | |
111 | } | |
112 | ||
113 | @@ -295,26 +289,45 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn, | |
114 | num_names += 1; | |
115 | } | |
116 | ||
117 | - tmp = TALLOC_REALLOC_ARRAY(mem_ctx, names, char *, num_names); | |
118 | - if (tmp == NULL) { | |
119 | + *pnum_names = num_names; | |
120 | + | |
121 | + if (pnames == NULL) { | |
122 | + TALLOC_FREE(to_free); | |
123 | + return NT_STATUS_OK; | |
124 | + } | |
125 | + | |
126 | + names = TALLOC_ARRAY(mem_ctx, char *, num_names); | |
127 | + if (names == NULL) { | |
128 | DEBUG(0, ("talloc failed\n")); | |
129 | - TALLOC_FREE(names); | |
130 | + TALLOC_FREE(to_free); | |
131 | return NT_STATUS_NO_MEMORY; | |
132 | } | |
133 | ||
134 | - names = tmp; | |
135 | + if (ea_namelist == smallbuf) { | |
136 | + ea_namelist = talloc_memdup(names, smallbuf, sizeret); | |
137 | + if (ea_namelist == NULL) { | |
138 | + TALLOC_FREE(names); | |
139 | + return NT_STATUS_NO_MEMORY; | |
140 | + } | |
141 | + } else { | |
142 | + talloc_steal(names, ea_namelist); | |
143 | + | |
144 | + ea_namelist = talloc_realloc(names, ea_namelist, char, | |
145 | + sizeret); | |
146 | + if (ea_namelist == NULL) { | |
147 | + TALLOC_FREE(names); | |
148 | + return NT_STATUS_NO_MEMORY; | |
149 | + } | |
150 | + } | |
151 | + | |
152 | num_names = 0; | |
153 | ||
154 | for (p = ea_namelist; p - ea_namelist < sizeret; p += strlen(p)+1) { | |
155 | names[num_names++] = p; | |
156 | } | |
157 | ||
158 | - if (pnames) { | |
159 | - *pnames = names; | |
160 | - } else { | |
161 | - TALLOC_FREE(names); | |
162 | - } | |
163 | - *pnum_names = num_names; | |
164 | + *pnames = names; | |
165 | + | |
166 | return NT_STATUS_OK; | |
167 | } | |
168 | ||
169 | -- | |
170 | 2.9.3 | |
171 | ||
172 | ||
173 | From 17563ab22ad19b34e1d9a1d12b2594c4186718b6 Mon Sep 17 00:00:00 2001 | |
174 | From: Volker Lendecke <vl@samba.org> | |
175 | Date: Tue, 25 Oct 2016 12:28:12 +0200 | |
176 | Subject: [PATCH 2/3] lib/util/charset: Optimize next_codepoint for the ascii | |
177 | case | |
178 | ||
179 | Signed-off-by: Volker Lendecke <vl@samba.org> | |
180 | Reviewed-by: Ralph Boehme <slow@samba.org> | |
181 | ||
182 | (cherry picked from commit 07d9a909ba6853fb0b96f6d86e4cf0d5d1b35b28) | |
183 | --- | |
184 | lib/util/charset/codepoints.c | 4 ++++ | |
185 | 1 file changed, 4 insertions(+) | |
186 | ||
187 | diff --git a/lib/util/charset/codepoints.c b/lib/util/charset/codepoints.c | |
188 | index 5ee95a8..7d157a3 100644 | |
189 | --- a/lib/util/charset/codepoints.c | |
190 | +++ b/lib/util/charset/codepoints.c | |
191 | @@ -495,6 +495,10 @@ _PUBLIC_ codepoint_t next_codepoint_ext(const char *str, charset_t src_charset, | |
192 | ||
193 | _PUBLIC_ codepoint_t next_codepoint(const char *str, size_t *size) | |
194 | { | |
195 | + if ((str[0] & 0x80) == 0) { | |
196 | + *size = 1; | |
197 | + return str[0]; | |
198 | + } | |
199 | return next_codepoint_convenience(get_iconv_convenience(), str, size); | |
200 | } | |
201 | ||
202 | -- | |
203 | 2.9.3 | |
204 | ||
205 | ||
206 | From ac8f6faa891fd282fb39ccb8e75a364bf97a5f2b Mon Sep 17 00:00:00 2001 | |
207 | From: Andreas Schneider <asn@samba.org> | |
208 | Date: Thu, 9 Feb 2017 15:05:01 +0100 | |
209 | Subject: [PATCH 3/3] s3-vfs: Only walk the directory once in | |
210 | open_and_sort_dir() | |
211 | ||
212 | On a slow filesystem or network filesystem this can make a huge | |
213 | difference. | |
214 | ||
215 | BUG: https://bugzilla.samba.org/show_bug.cgi?id=12571 | |
216 | ||
217 | Signed-off-by: Andreas Schneider <asn@samba.org> | |
218 | --- | |
219 | source3/modules/vfs_dirsort.c | 61 +++++++++++++++++++++++++++---------------- | |
220 | 1 file changed, 38 insertions(+), 23 deletions(-) | |
221 | ||
222 | diff --git a/source3/modules/vfs_dirsort.c b/source3/modules/vfs_dirsort.c | |
223 | index 698e96b..66582e6 100644 | |
224 | --- a/source3/modules/vfs_dirsort.c | |
225 | +++ b/source3/modules/vfs_dirsort.c | |
226 | @@ -68,8 +68,10 @@ static bool get_sorted_dir_mtime(vfs_handle_struct *handle, | |
227 | static bool open_and_sort_dir(vfs_handle_struct *handle, | |
228 | struct dirsort_privates *data) | |
229 | { | |
230 | - unsigned int i = 0; | |
231 | - unsigned int total_count = 0; | |
232 | + uint32_t total_count = 0; | |
233 | + /* This should be enough for most use cases */ | |
234 | + uint32_t dirent_allocated = 64; | |
235 | + SMB_STRUCT_DIRENT *dp; | |
236 | ||
237 | data->number_of_entries = 0; | |
238 | ||
239 | @@ -77,38 +79,51 @@ static bool open_and_sort_dir(vfs_handle_struct *handle, | |
240 | return false; | |
241 | } | |
242 | ||
243 | - while (SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL) | |
244 | - != NULL) { | |
245 | - total_count++; | |
246 | - } | |
247 | - | |
248 | - if (total_count == 0) { | |
249 | + dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL); | |
250 | + if (dp == NULL) { | |
251 | return false; | |
252 | } | |
253 | ||
254 | - /* Open the underlying directory and count the number of entries | |
255 | - Skip back to the beginning as we'll read it again */ | |
256 | - SMB_VFS_NEXT_REWINDDIR(handle, data->source_directory); | |
257 | - | |
258 | /* Set up an array and read the directory entries into it */ | |
259 | TALLOC_FREE(data->directory_list); /* destroy previous cache if needed */ | |
260 | data->directory_list = talloc_zero_array(data, | |
261 | SMB_STRUCT_DIRENT, | |
262 | - total_count); | |
263 | - if (!data->directory_list) { | |
264 | + dirent_allocated); | |
265 | + if (data->directory_list == NULL) { | |
266 | return false; | |
267 | } | |
268 | - for (i = 0; i < total_count; i++) { | |
269 | - SMB_STRUCT_DIRENT *dp = SMB_VFS_NEXT_READDIR(handle, | |
270 | - data->source_directory, | |
271 | - NULL); | |
272 | - if (dp == NULL) { | |
273 | - break; | |
274 | + | |
275 | + do { | |
276 | + if (total_count >= dirent_allocated) { | |
277 | + struct dirent *dlist; | |
278 | + | |
279 | + /* | |
280 | + * Be memory friendly. | |
281 | + * | |
282 | + * We should not double the amount of memory. With a lot | |
283 | + * of files we reach easily 50MB, and doubling will | |
284 | + * get much bigger just for a few files more. | |
285 | + * | |
286 | + * For 200k files this means 50 memory reallocations. | |
287 | + */ | |
288 | + dirent_allocated += 4096; | |
289 | + | |
290 | + dlist = talloc_realloc(data, | |
291 | + data->directory_list, | |
292 | + SMB_STRUCT_DIRENT, | |
293 | + dirent_allocated); | |
294 | + if (dlist == NULL) { | |
295 | + break; | |
296 | + } | |
297 | + data->directory_list = dlist; | |
298 | } | |
299 | - data->directory_list[i] = *dp; | |
300 | - } | |
301 | + data->directory_list[total_count] = *dp; | |
302 | + | |
303 | + total_count++; | |
304 | + dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL); | |
305 | + } while (dp != NULL); | |
306 | ||
307 | - data->number_of_entries = i; | |
308 | + data->number_of_entries = total_count; | |
309 | ||
310 | /* Sort the directory entries by name */ | |
311 | TYPESAFE_QSORT(data->directory_list, data->number_of_entries, compare_dirent); | |
312 | -- | |
313 | 2.9.3 | |
314 |