1 From 252499c1513c45764d039af8732cd97b37c8c494 Mon Sep 17 00:00:00 2001
2 From: Volker Lendecke <vl@samba.org>
3 Date: Thu, 9 Feb 2017 15:40:39 +0100
4 Subject: [PATCH 1/3] smbd: Streamline get_ea_names_from_file
6 Signed-off-by: Volker Lendecke <vl@samba.org>
7 Reviewed-by: Ralph Boehme <slow@samba.org>
8 Backported-by: Andreas Schneider <asn@samba.org>
9 Backported-from: 27daed8fcf95eed2df112dc1c30c3a40b5c9565b
11 source3/smbd/trans2.c | 89 +++++++++++++++++++++++++++++----------------------
12 1 file changed, 51 insertions(+), 38 deletions(-)
14 diff --git a/source3/smbd/trans2.c b/source3/smbd/trans2.c
15 index 98fd2af..49cfe9f 100644
16 --- a/source3/smbd/trans2.c
17 +++ b/source3/smbd/trans2.c
18 @@ -201,12 +201,14 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
19 files_struct *fsp, const char *fname,
20 char ***pnames, size_t *pnum_names)
22 + char smallbuf[1024];
23 /* Get a list of all xattrs. Max namesize is 64k. */
24 size_t ea_namelist_size = 1024;
25 - char *ea_namelist = NULL;
26 + char *ea_namelist = smallbuf;
27 + char *to_free = NULL;
30 - char **names, **tmp;
35 @@ -228,25 +230,24 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
40 - * TALLOC the result early to get the talloc hierarchy right.
43 - names = TALLOC_ARRAY(mem_ctx, char *, 1);
44 - if (names == NULL) {
45 - DEBUG(0, ("talloc failed\n"));
46 - return NT_STATUS_NO_MEMORY;
47 + if (fsp && fsp->fh->fd != -1) {
48 + sizeret = SMB_VFS_FLISTXATTR(fsp, ea_namelist,
51 + sizeret = SMB_VFS_LISTXATTR(conn,
57 - while (ea_namelist_size <= 65536) {
59 - ea_namelist = TALLOC_REALLOC_ARRAY(
60 - names, ea_namelist, char, ea_namelist_size);
61 + if ((sizeret == -1) && (errno == ERANGE)) {
62 + ea_namelist_size = 65536;
63 + ea_namelist = TALLOC_ARRAY(mem_ctx, char, ea_namelist_size);
64 if (ea_namelist == NULL) {
65 DEBUG(0, ("talloc failed\n"));
67 return NT_STATUS_NO_MEMORY;
69 + to_free = ea_namelist;
71 if (fsp && fsp->fh->fd != -1) {
72 sizeret = SMB_VFS_FLISTXATTR(fsp, ea_namelist,
73 @@ -255,25 +256,18 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
74 sizeret = SMB_VFS_LISTXATTR(conn, fname, ea_namelist,
78 - if ((sizeret == -1) && (errno == ERANGE)) {
79 - ea_namelist_size *= 2;
88 - return map_nt_error_from_unix(errno);
89 + status = map_nt_error_from_unix(errno);
90 + TALLOC_FREE(to_free);
94 - DEBUG(10, ("get_ea_list_from_file: ea_namelist size = %u\n",
95 - (unsigned int)sizeret));
96 + DEBUG(10, ("ea_namelist size = %zd\n", sizeret));
100 + TALLOC_FREE(to_free);
104 @@ -282,7 +276,7 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
107 if (ea_namelist[sizeret-1] != '\0') {
108 - TALLOC_FREE(names);
109 + TALLOC_FREE(to_free);
110 return NT_STATUS_INTERNAL_ERROR;
113 @@ -295,26 +289,45 @@ NTSTATUS get_ea_names_from_file(TALLOC_CTX *mem_ctx, connection_struct *conn,
117 - tmp = TALLOC_REALLOC_ARRAY(mem_ctx, names, char *, num_names);
119 + *pnum_names = num_names;
121 + if (pnames == NULL) {
122 + TALLOC_FREE(to_free);
123 + return NT_STATUS_OK;
126 + names = TALLOC_ARRAY(mem_ctx, char *, num_names);
127 + if (names == NULL) {
128 DEBUG(0, ("talloc failed\n"));
129 - TALLOC_FREE(names);
130 + TALLOC_FREE(to_free);
131 return NT_STATUS_NO_MEMORY;
135 + if (ea_namelist == smallbuf) {
136 + ea_namelist = talloc_memdup(names, smallbuf, sizeret);
137 + if (ea_namelist == NULL) {
138 + TALLOC_FREE(names);
139 + return NT_STATUS_NO_MEMORY;
142 + talloc_steal(names, ea_namelist);
144 + ea_namelist = talloc_realloc(names, ea_namelist, char,
146 + if (ea_namelist == NULL) {
147 + TALLOC_FREE(names);
148 + return NT_STATUS_NO_MEMORY;
154 for (p = ea_namelist; p - ea_namelist < sizeret; p += strlen(p)+1) {
155 names[num_names++] = p;
161 - TALLOC_FREE(names);
163 - *pnum_names = num_names;
173 From 17563ab22ad19b34e1d9a1d12b2594c4186718b6 Mon Sep 17 00:00:00 2001
174 From: Volker Lendecke <vl@samba.org>
175 Date: Tue, 25 Oct 2016 12:28:12 +0200
176 Subject: [PATCH 2/3] lib/util/charset: Optimize next_codepoint for the ascii
179 Signed-off-by: Volker Lendecke <vl@samba.org>
180 Reviewed-by: Ralph Boehme <slow@samba.org>
182 (cherry picked from commit 07d9a909ba6853fb0b96f6d86e4cf0d5d1b35b28)
184 lib/util/charset/codepoints.c | 4 ++++
185 1 file changed, 4 insertions(+)
187 diff --git a/lib/util/charset/codepoints.c b/lib/util/charset/codepoints.c
188 index 5ee95a8..7d157a3 100644
189 --- a/lib/util/charset/codepoints.c
190 +++ b/lib/util/charset/codepoints.c
191 @@ -495,6 +495,10 @@ _PUBLIC_ codepoint_t next_codepoint_ext(const char *str, charset_t src_charset,
193 _PUBLIC_ codepoint_t next_codepoint(const char *str, size_t *size)
195 + if ((str[0] & 0x80) == 0) {
199 return next_codepoint_convenience(get_iconv_convenience(), str, size);
206 From ac8f6faa891fd282fb39ccb8e75a364bf97a5f2b Mon Sep 17 00:00:00 2001
207 From: Andreas Schneider <asn@samba.org>
208 Date: Thu, 9 Feb 2017 15:05:01 +0100
209 Subject: [PATCH 3/3] s3-vfs: Only walk the directory once in
212 On a slow filesystem or network filesystem this can make a huge
215 BUG: https://bugzilla.samba.org/show_bug.cgi?id=12571
217 Signed-off-by: Andreas Schneider <asn@samba.org>
219 source3/modules/vfs_dirsort.c | 61 +++++++++++++++++++++++++++----------------
220 1 file changed, 38 insertions(+), 23 deletions(-)
222 diff --git a/source3/modules/vfs_dirsort.c b/source3/modules/vfs_dirsort.c
223 index 698e96b..66582e6 100644
224 --- a/source3/modules/vfs_dirsort.c
225 +++ b/source3/modules/vfs_dirsort.c
226 @@ -68,8 +68,10 @@ static bool get_sorted_dir_mtime(vfs_handle_struct *handle,
227 static bool open_and_sort_dir(vfs_handle_struct *handle,
228 struct dirsort_privates *data)
230 - unsigned int i = 0;
231 - unsigned int total_count = 0;
232 + uint32_t total_count = 0;
233 + /* This should be enough for most use cases */
234 + uint32_t dirent_allocated = 64;
235 + SMB_STRUCT_DIRENT *dp;
237 data->number_of_entries = 0;
239 @@ -77,38 +79,51 @@ static bool open_and_sort_dir(vfs_handle_struct *handle,
243 - while (SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL)
248 - if (total_count == 0) {
249 + dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
254 - /* Open the underlying directory and count the number of entries
255 - Skip back to the beginning as we'll read it again */
256 - SMB_VFS_NEXT_REWINDDIR(handle, data->source_directory);
258 /* Set up an array and read the directory entries into it */
259 TALLOC_FREE(data->directory_list); /* destroy previous cache if needed */
260 data->directory_list = talloc_zero_array(data,
263 - if (!data->directory_list) {
265 + if (data->directory_list == NULL) {
268 - for (i = 0; i < total_count; i++) {
269 - SMB_STRUCT_DIRENT *dp = SMB_VFS_NEXT_READDIR(handle,
270 - data->source_directory,
276 + if (total_count >= dirent_allocated) {
277 + struct dirent *dlist;
280 + * Be memory friendly.
282 + * We should not double the amount of memory. With a lot
283 + * of files we reach easily 50MB, and doubling will
284 + * get much bigger just for a few files more.
286 + * For 200k files this means 50 memory reallocations.
288 + dirent_allocated += 4096;
290 + dlist = talloc_realloc(data,
291 + data->directory_list,
294 + if (dlist == NULL) {
297 + data->directory_list = dlist;
299 - data->directory_list[i] = *dp;
301 + data->directory_list[total_count] = *dp;
304 + dp = SMB_VFS_NEXT_READDIR(handle, data->source_directory, NULL);
305 + } while (dp != NULL);
307 - data->number_of_entries = i;
308 + data->number_of_entries = total_count;
310 /* Sort the directory entries by name */
311 TYPESAFE_QSORT(data->directory_list, data->number_of_entries, compare_dirent);