]>
Commit | Line | Data |
---|---|---|
cbdee279 | 1 | /* Handle general operations. |
04277e02 | 2 | Copyright (C) 1997-2019 Free Software Foundation, Inc. |
cbdee279 UD |
3 | This file is part of the GNU C Library. |
4 | Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997. | |
5 | ||
6 | The GNU C Library is free software; you can redistribute it and/or | |
41bdb6e2 AJ |
7 | modify it under the terms of the GNU Lesser General Public |
8 | License as published by the Free Software Foundation; either | |
9 | version 2.1 of the License, or (at your option) any later version. | |
cbdee279 UD |
10 | |
11 | The GNU C Library is distributed in the hope that it will be useful, | |
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
41bdb6e2 | 14 | Lesser General Public License for more details. |
cbdee279 | 15 | |
41bdb6e2 | 16 | You should have received a copy of the GNU Lesser General Public |
59ba27a6 | 17 | License along with the GNU C Library; if not, see |
5a82c748 | 18 | <https://www.gnu.org/licenses/>. */ |
cbdee279 UD |
19 | |
20 | #include <aio.h> | |
a3bfd999 | 21 | #include <assert.h> |
cbdee279 | 22 | #include <errno.h> |
4959e310 | 23 | #include <limits.h> |
cbdee279 | 24 | #include <pthread.h> |
cbdee279 UD |
25 | #include <stdlib.h> |
26 | #include <unistd.h> | |
2fc54d6f | 27 | #include <sys/param.h> |
cbdee279 | 28 | #include <sys/stat.h> |
a3bfd999 | 29 | #include <sys/time.h> |
ffdd5e50 | 30 | #include <aio_misc.h> |
cbdee279 | 31 | |
ffdd5e50 UD |
32 | #ifndef aio_create_helper_thread |
33 | # define aio_create_helper_thread __aio_create_helper_thread | |
34 | ||
35 | extern inline int | |
36 | __aio_create_helper_thread (pthread_t *threadp, void *(*tf) (void *), void *arg) | |
37 | { | |
38 | pthread_attr_t attr; | |
39 | ||
40 | /* Make sure the thread is created detached. */ | |
41 | pthread_attr_init (&attr); | |
42 | pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED); | |
43 | ||
44 | int ret = pthread_create (threadp, &attr, tf, arg); | |
45 | ||
46 | (void) pthread_attr_destroy (&attr); | |
47 | return ret; | |
6d3aff23 | 48 | } |
ffdd5e50 | 49 | #endif |
cbdee279 | 50 | |
a3bfd999 UD |
51 | static void add_request_to_runlist (struct requestlist *newrequest); |
52 | ||
d71b808a UD |
53 | /* Pool of request list entries. */ |
54 | static struct requestlist **pool; | |
cbdee279 | 55 | |
d71b808a | 56 | /* Number of total and allocated pool entries. */ |
56ddf355 | 57 | static size_t pool_max_size; |
d71b808a | 58 | static size_t pool_size; |
cbdee279 | 59 | |
d71b808a UD |
60 | /* We implement a two dimensional array but allocate each row separately. |
61 | The macro below determines how many entries should be used per row. | |
62 | It should better be a power of two. */ | |
56ddf355 | 63 | #define ENTRIES_PER_ROW 32 |
cbdee279 | 64 | |
56ddf355 UD |
65 | /* How many rows we allocate at once. */ |
66 | #define ROWS_STEP 8 | |
d71b808a UD |
67 | |
68 | /* List of available entries. */ | |
69 | static struct requestlist *freelist; | |
70 | ||
71 | /* List of request waiting to be processed. */ | |
72 | static struct requestlist *runlist; | |
73 | ||
74 | /* Structure list of all currently processed requests. */ | |
75 | static struct requestlist *requests; | |
76 | ||
77 | /* Number of threads currently running. */ | |
78 | static int nthreads; | |
79 | ||
a3bfd999 UD |
80 | /* Number of threads waiting for work to arrive. */ |
81 | static int idle_thread_count; | |
82 | ||
d71b808a UD |
83 | |
84 | /* These are the values used to optimize the use of AIO. The user can | |
85 | overwrite them by using the `aio_init' function. */ | |
86 | static struct aioinit optim = | |
87 | { | |
88 | 20, /* int aio_threads; Maximal number of threads. */ | |
2fc54d6f | 89 | 64, /* int aio_num; Number of expected simultaneous requests. */ |
d71b808a UD |
90 | 0, |
91 | 0, | |
92 | 0, | |
93 | 0, | |
a3bfd999 UD |
94 | 1, |
95 | 0 | |
d71b808a UD |
96 | }; |
97 | ||
98 | ||
99 | /* Since the list is global we need a mutex protecting it. */ | |
100 | pthread_mutex_t __aio_requests_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; | |
101 | ||
a3bfd999 UD |
102 | /* When you add a request to the list and there are idle threads present, |
103 | you signal this condition variable. When a thread finishes work, it waits | |
104 | on this condition variable for a time before it actually exits. */ | |
105 | pthread_cond_t __aio_new_request_notification = PTHREAD_COND_INITIALIZER; | |
106 | ||
d71b808a UD |
107 | |
108 | /* Functions to handle request list pool. */ | |
109 | static struct requestlist * | |
110 | get_elem (void) | |
cbdee279 | 111 | { |
d71b808a UD |
112 | struct requestlist *result; |
113 | ||
114 | if (freelist == NULL) | |
115 | { | |
116 | struct requestlist *new_row; | |
56ddf355 | 117 | int cnt; |
d71b808a | 118 | |
b6aa34eb | 119 | assert (sizeof (struct aiocb) == sizeof (struct aiocb64)); |
a3bfd999 | 120 | |
56ddf355 | 121 | if (pool_size + 1 >= pool_max_size) |
d71b808a | 122 | { |
56ddf355 | 123 | size_t new_max_size = pool_max_size + ROWS_STEP; |
d71b808a UD |
124 | struct requestlist **new_tab; |
125 | ||
126 | new_tab = (struct requestlist **) | |
56ddf355 | 127 | realloc (pool, new_max_size * sizeof (struct requestlist *)); |
d71b808a UD |
128 | |
129 | if (new_tab == NULL) | |
130 | return NULL; | |
131 | ||
56ddf355 | 132 | pool_max_size = new_max_size; |
d71b808a UD |
133 | pool = new_tab; |
134 | } | |
135 | ||
2ace5721 | 136 | /* Allocate the new row. */ |
56ddf355 UD |
137 | cnt = pool_size == 0 ? optim.aio_num : ENTRIES_PER_ROW; |
138 | new_row = (struct requestlist *) calloc (cnt, | |
139 | sizeof (struct requestlist)); | |
140 | if (new_row == NULL) | |
141 | return NULL; | |
d71b808a | 142 | |
56ddf355 | 143 | pool[pool_size++] = new_row; |
d71b808a UD |
144 | |
145 | /* Put all the new entries in the freelist. */ | |
146 | do | |
147 | { | |
148 | new_row->next_prio = freelist; | |
149 | freelist = new_row++; | |
150 | } | |
56ddf355 | 151 | while (--cnt > 0); |
d71b808a UD |
152 | } |
153 | ||
154 | result = freelist; | |
155 | freelist = freelist->next_prio; | |
156 | ||
157 | return result; | |
cbdee279 UD |
158 | } |
159 | ||
d71b808a UD |
160 | |
161 | void | |
bd355af0 | 162 | __aio_free_request (struct requestlist *elem) |
d71b808a UD |
163 | { |
164 | elem->running = no; | |
165 | elem->next_prio = freelist; | |
166 | freelist = elem; | |
167 | } | |
168 | ||
169 | ||
170 | struct requestlist * | |
171 | __aio_find_req (aiocb_union *elem) | |
172 | { | |
173 | struct requestlist *runp = requests; | |
174 | int fildes = elem->aiocb.aio_fildes; | |
175 | ||
176 | while (runp != NULL && runp->aiocbp->aiocb.aio_fildes < fildes) | |
177 | runp = runp->next_fd; | |
178 | ||
179 | if (runp != NULL) | |
7ef90c15 UD |
180 | { |
181 | if (runp->aiocbp->aiocb.aio_fildes != fildes) | |
182 | runp = NULL; | |
183 | else | |
184 | while (runp != NULL && runp->aiocbp != elem) | |
185 | runp = runp->next_prio; | |
186 | } | |
d71b808a UD |
187 | |
188 | return runp; | |
189 | } | |
190 | ||
191 | ||
192 | struct requestlist * | |
193 | __aio_find_req_fd (int fildes) | |
194 | { | |
195 | struct requestlist *runp = requests; | |
196 | ||
197 | while (runp != NULL && runp->aiocbp->aiocb.aio_fildes < fildes) | |
198 | runp = runp->next_fd; | |
199 | ||
200 | return (runp != NULL && runp->aiocbp->aiocb.aio_fildes == fildes | |
201 | ? runp : NULL); | |
202 | } | |
cbdee279 UD |
203 | |
204 | ||
92806ee9 | 205 | void |
92806ee9 UD |
206 | __aio_remove_request (struct requestlist *last, struct requestlist *req, |
207 | int all) | |
208 | { | |
56ddf355 UD |
209 | assert (req->running == yes || req->running == queued |
210 | || req->running == done); | |
211 | ||
92806ee9 | 212 | if (last != NULL) |
56ddf355 | 213 | last->next_prio = all ? NULL : req->next_prio; |
92806ee9 UD |
214 | else |
215 | { | |
216 | if (all || req->next_prio == NULL) | |
217 | { | |
218 | if (req->last_fd != NULL) | |
219 | req->last_fd->next_fd = req->next_fd; | |
220 | else | |
221 | requests = req->next_fd; | |
222 | if (req->next_fd != NULL) | |
223 | req->next_fd->last_fd = req->last_fd; | |
224 | } | |
225 | else | |
226 | { | |
227 | if (req->last_fd != NULL) | |
228 | req->last_fd->next_fd = req->next_prio; | |
229 | else | |
230 | requests = req->next_prio; | |
231 | ||
232 | if (req->next_fd != NULL) | |
233 | req->next_fd->last_fd = req->next_prio; | |
234 | ||
235 | req->next_prio->last_fd = req->last_fd; | |
236 | req->next_prio->next_fd = req->next_fd; | |
237 | ||
238 | /* Mark this entry as runnable. */ | |
239 | req->next_prio->running = yes; | |
240 | } | |
241 | ||
242 | if (req->running == yes) | |
243 | { | |
244 | struct requestlist *runp = runlist; | |
245 | ||
246 | last = NULL; | |
247 | while (runp != NULL) | |
248 | { | |
249 | if (runp == req) | |
250 | { | |
251 | if (last == NULL) | |
252 | runlist = runp->next_run; | |
253 | else | |
254 | last->next_run = runp->next_run; | |
255 | break; | |
256 | } | |
257 | last = runp; | |
258 | runp = runp->next_run; | |
259 | } | |
260 | } | |
261 | } | |
262 | } | |
263 | ||
264 | ||
cbdee279 UD |
265 | /* The thread handler. */ |
266 | static void *handle_fildes_io (void *arg); | |
267 | ||
268 | ||
d71b808a UD |
269 | /* User optimization. */ |
270 | void | |
271 | __aio_init (const struct aioinit *init) | |
272 | { | |
273 | /* Get the mutex. */ | |
274 | pthread_mutex_lock (&__aio_requests_mutex); | |
275 | ||
276 | /* Only allow writing new values if the table is not yet allocated. */ | |
277 | if (pool == NULL) | |
278 | { | |
279 | optim.aio_threads = init->aio_threads < 1 ? 1 : init->aio_threads; | |
2fc54d6f | 280 | assert (powerof2 (ENTRIES_PER_ROW)); |
d71b808a UD |
281 | optim.aio_num = (init->aio_num < ENTRIES_PER_ROW |
282 | ? ENTRIES_PER_ROW | |
2fc54d6f | 283 | : init->aio_num & ~(ENTRIES_PER_ROW - 1)); |
d71b808a UD |
284 | } |
285 | ||
a3bfd999 UD |
286 | if (init->aio_idle_time != 0) |
287 | optim.aio_idle_time = init->aio_idle_time; | |
288 | ||
d71b808a UD |
289 | /* Release the mutex. */ |
290 | pthread_mutex_unlock (&__aio_requests_mutex); | |
291 | } | |
292 | weak_alias (__aio_init, aio_init) | |
293 | ||
294 | ||
cbdee279 UD |
295 | /* The main function of the async I/O handling. It enqueues requests |
296 | and if necessary starts and handles threads. */ | |
d71b808a UD |
297 | struct requestlist * |
298 | __aio_enqueue_request (aiocb_union *aiocbp, int operation) | |
cbdee279 | 299 | { |
d71b808a | 300 | int result = 0; |
cbdee279 UD |
301 | int policy, prio; |
302 | struct sched_param param; | |
d71b808a UD |
303 | struct requestlist *last, *runp, *newp; |
304 | int running = no; | |
cbdee279 | 305 | |
92806ee9 UD |
306 | if (operation == LIO_SYNC || operation == LIO_DSYNC) |
307 | aiocbp->aiocb.aio_reqprio = 0; | |
308 | else if (aiocbp->aiocb.aio_reqprio < 0 | |
e2049d17 ST |
309 | #ifdef AIO_PRIO_DELTA_MAX |
310 | || aiocbp->aiocb.aio_reqprio > AIO_PRIO_DELTA_MAX | |
311 | #endif | |
312 | ) | |
cbdee279 UD |
313 | { |
314 | /* Invalid priority value. */ | |
315 | __set_errno (EINVAL); | |
316 | aiocbp->aiocb.__error_code = EINVAL; | |
317 | aiocbp->aiocb.__return_value = -1; | |
d71b808a | 318 | return NULL; |
cbdee279 UD |
319 | } |
320 | ||
321 | /* Compute priority for this request. */ | |
d71b808a | 322 | pthread_getschedparam (pthread_self (), &policy, ¶m); |
cbdee279 UD |
323 | prio = param.sched_priority - aiocbp->aiocb.aio_reqprio; |
324 | ||
d71b808a UD |
325 | /* Get the mutex. */ |
326 | pthread_mutex_lock (&__aio_requests_mutex); | |
cbdee279 | 327 | |
d71b808a UD |
328 | last = NULL; |
329 | runp = requests; | |
cbdee279 UD |
330 | /* First look whether the current file descriptor is currently |
331 | worked with. */ | |
d71b808a UD |
332 | while (runp != NULL |
333 | && runp->aiocbp->aiocb.aio_fildes < aiocbp->aiocb.aio_fildes) | |
334 | { | |
335 | last = runp; | |
336 | runp = runp->next_fd; | |
337 | } | |
cbdee279 | 338 | |
d71b808a UD |
339 | /* Get a new element for the waiting list. */ |
340 | newp = get_elem (); | |
341 | if (newp == NULL) | |
342 | { | |
d71b808a | 343 | pthread_mutex_unlock (&__aio_requests_mutex); |
4bea7a2e | 344 | __set_errno (EAGAIN); |
d71b808a UD |
345 | return NULL; |
346 | } | |
347 | newp->aiocbp = aiocbp; | |
348 | newp->waiting = NULL; | |
349 | ||
350 | aiocbp->aiocb.__abs_prio = prio; | |
351 | aiocbp->aiocb.__policy = policy; | |
352 | aiocbp->aiocb.aio_lio_opcode = operation; | |
353 | aiocbp->aiocb.__error_code = EINPROGRESS; | |
354 | aiocbp->aiocb.__return_value = 0; | |
355 | ||
356 | if (runp != NULL | |
357 | && runp->aiocbp->aiocb.aio_fildes == aiocbp->aiocb.aio_fildes) | |
cbdee279 UD |
358 | { |
359 | /* The current file descriptor is worked on. It makes no sense | |
d71b808a UD |
360 | to start another thread since this new thread would fight |
361 | with the running thread for the resources. But we also cannot | |
b9337b6a | 362 | say that the thread processing this desriptor shall immediately |
d71b808a UD |
363 | after finishing the current job process this request if there |
364 | are other threads in the running queue which have a higher | |
365 | priority. */ | |
366 | ||
367 | /* Simply enqueue it after the running one according to the | |
368 | priority. */ | |
584715c3 | 369 | last = NULL; |
d71b808a UD |
370 | while (runp->next_prio != NULL |
371 | && runp->next_prio->aiocbp->aiocb.__abs_prio >= prio) | |
584715c3 UD |
372 | { |
373 | last = runp; | |
374 | runp = runp->next_prio; | |
375 | } | |
d71b808a UD |
376 | |
377 | newp->next_prio = runp->next_prio; | |
378 | runp->next_prio = newp; | |
379 | ||
380 | running = queued; | |
cbdee279 UD |
381 | } |
382 | else | |
383 | { | |
a3bfd999 | 384 | running = yes; |
d71b808a UD |
385 | /* Enqueue this request for a new descriptor. */ |
386 | if (last == NULL) | |
387 | { | |
388 | newp->last_fd = NULL; | |
389 | newp->next_fd = requests; | |
390 | if (requests != NULL) | |
391 | requests->last_fd = newp; | |
392 | requests = newp; | |
393 | } | |
394 | else | |
395 | { | |
396 | newp->next_fd = last->next_fd; | |
397 | newp->last_fd = last; | |
398 | last->next_fd = newp; | |
399 | if (newp->next_fd != NULL) | |
400 | newp->next_fd->last_fd = newp; | |
401 | } | |
402 | ||
403 | newp->next_prio = NULL; | |
584715c3 | 404 | last = NULL; |
d71b808a UD |
405 | } |
406 | ||
a3bfd999 | 407 | if (running == yes) |
d71b808a UD |
408 | { |
409 | /* We try to create a new thread for this file descriptor. The | |
cbdee279 UD |
410 | function which gets called will handle all available requests |
411 | for this descriptor and when all are processed it will | |
d71b808a UD |
412 | terminate. |
413 | ||
414 | If no new thread can be created or if the specified limit of | |
415 | threads for AIO is reached we queue the request. */ | |
416 | ||
a3bfd999 UD |
417 | /* See if we need to and are able to create a thread. */ |
418 | if (nthreads < optim.aio_threads && idle_thread_count == 0) | |
cbdee279 | 419 | { |
d71b808a | 420 | pthread_t thid; |
d71b808a | 421 | |
56ddf355 UD |
422 | running = newp->running = allocated; |
423 | ||
d71b808a | 424 | /* Now try to start a thread. */ |
584715c3 UD |
425 | result = aio_create_helper_thread (&thid, handle_fildes_io, newp); |
426 | if (result == 0) | |
56ddf355 UD |
427 | /* We managed to enqueue the request. All errors which can |
428 | happen now can be recognized by calls to `aio_return' and | |
429 | `aio_error'. */ | |
430 | ++nthreads; | |
431 | else | |
d71b808a | 432 | { |
2ace5721 | 433 | /* Reset the running flag. The new request is not running. */ |
56ddf355 UD |
434 | running = newp->running = yes; |
435 | ||
436 | if (nthreads == 0) | |
584715c3 UD |
437 | { |
438 | /* We cannot create a thread in the moment and there is | |
439 | also no thread running. This is a problem. `errno' is | |
440 | set to EAGAIN if this is only a temporary problem. */ | |
441 | __aio_remove_request (last, newp, 0); | |
442 | } | |
443 | else | |
444 | result = 0; | |
d71b808a | 445 | } |
d71b808a UD |
446 | } |
447 | } | |
448 | ||
449 | /* Enqueue the request in the run queue if it is not yet running. */ | |
a3bfd999 | 450 | if (running == yes && result == 0) |
d71b808a | 451 | { |
a3bfd999 | 452 | add_request_to_runlist (newp); |
d71b808a | 453 | |
a3bfd999 UD |
454 | /* If there is a thread waiting for work, then let it know that we |
455 | have just given it something to do. */ | |
456 | if (idle_thread_count > 0) | |
457 | pthread_cond_signal (&__aio_new_request_notification); | |
cbdee279 UD |
458 | } |
459 | ||
04794f3e ST |
460 | if (result == 0) |
461 | newp->running = running; | |
462 | else | |
d71b808a UD |
463 | { |
464 | /* Something went wrong. */ | |
bd355af0 | 465 | __aio_free_request (newp); |
584715c3 UD |
466 | aiocbp->aiocb.__error_code = result; |
467 | __set_errno (result); | |
d71b808a UD |
468 | newp = NULL; |
469 | } | |
cbdee279 | 470 | |
d71b808a UD |
471 | /* Release the mutex. */ |
472 | pthread_mutex_unlock (&__aio_requests_mutex); | |
473 | ||
474 | return newp; | |
cbdee279 UD |
475 | } |
476 | ||
477 | ||
478 | static void * | |
479 | handle_fildes_io (void *arg) | |
480 | { | |
481 | pthread_t self = pthread_self (); | |
482 | struct sched_param param; | |
d71b808a | 483 | struct requestlist *runp = (struct requestlist *) arg; |
b9337b6a | 484 | aiocb_union *aiocbp; |
cbdee279 | 485 | int policy; |
b9337b6a | 486 | int fildes; |
cbdee279 UD |
487 | |
488 | pthread_getschedparam (self, &policy, ¶m); | |
489 | ||
490 | do | |
491 | { | |
a3bfd999 UD |
492 | /* If runp is NULL, then we were created to service the work queue |
493 | in general, not to handle any particular request. In that case we | |
494 | skip the "do work" stuff on the first pass, and go directly to the | |
495 | "get work off the work queue" part of this loop, which is near the | |
496 | end. */ | |
497 | if (runp == NULL) | |
498 | pthread_mutex_lock (&__aio_requests_mutex); | |
499 | else | |
cbdee279 | 500 | { |
56ddf355 UD |
501 | /* Hopefully this request is marked as running. */ |
502 | assert (runp->running == allocated); | |
503 | ||
a3bfd999 UD |
504 | /* Update our variables. */ |
505 | aiocbp = runp->aiocbp; | |
506 | fildes = aiocbp->aiocb.aio_fildes; | |
cbdee279 | 507 | |
a3bfd999 UD |
508 | /* Change the priority to the requested value (if necessary). */ |
509 | if (aiocbp->aiocb.__abs_prio != param.sched_priority | |
510 | || aiocbp->aiocb.__policy != policy) | |
511 | { | |
512 | param.sched_priority = aiocbp->aiocb.__abs_prio; | |
513 | policy = aiocbp->aiocb.__policy; | |
514 | pthread_setschedparam (self, policy, ¶m); | |
515 | } | |
516 | ||
517 | /* Process request pointed to by RUNP. We must not be disturbed | |
518 | by signals. */ | |
519 | if ((aiocbp->aiocb.aio_lio_opcode & 127) == LIO_READ) | |
520 | { | |
01e7fdbb UD |
521 | if (sizeof (off_t) != sizeof (off64_t) |
522 | && aiocbp->aiocb.aio_lio_opcode & 128) | |
a3bfd999 UD |
523 | aiocbp->aiocb.__return_value = |
524 | TEMP_FAILURE_RETRY (__pread64 (fildes, (void *) | |
525 | aiocbp->aiocb64.aio_buf, | |
526 | aiocbp->aiocb64.aio_nbytes, | |
527 | aiocbp->aiocb64.aio_offset)); | |
528 | else | |
529 | aiocbp->aiocb.__return_value = | |
9714012c JM |
530 | TEMP_FAILURE_RETRY (__libc_pread (fildes, |
531 | (void *) | |
532 | aiocbp->aiocb.aio_buf, | |
533 | aiocbp->aiocb.aio_nbytes, | |
534 | aiocbp->aiocb.aio_offset)); | |
a3bfd999 UD |
535 | |
536 | if (aiocbp->aiocb.__return_value == -1 && errno == ESPIPE) | |
537 | /* The Linux kernel is different from others. It returns | |
538 | ESPIPE if using pread on a socket. Other platforms | |
539 | simply ignore the offset parameter and behave like | |
540 | read. */ | |
541 | aiocbp->aiocb.__return_value = | |
542 | TEMP_FAILURE_RETRY (read (fildes, | |
543 | (void *) aiocbp->aiocb64.aio_buf, | |
544 | aiocbp->aiocb64.aio_nbytes)); | |
545 | } | |
546 | else if ((aiocbp->aiocb.aio_lio_opcode & 127) == LIO_WRITE) | |
547 | { | |
01e7fdbb UD |
548 | if (sizeof (off_t) != sizeof (off64_t) |
549 | && aiocbp->aiocb.aio_lio_opcode & 128) | |
a3bfd999 UD |
550 | aiocbp->aiocb.__return_value = |
551 | TEMP_FAILURE_RETRY (__pwrite64 (fildes, (const void *) | |
552 | aiocbp->aiocb64.aio_buf, | |
553 | aiocbp->aiocb64.aio_nbytes, | |
554 | aiocbp->aiocb64.aio_offset)); | |
555 | else | |
556 | aiocbp->aiocb.__return_value = | |
c877418f | 557 | TEMP_FAILURE_RETRY (__libc_pwrite (fildes, (const void *) |
a3bfd999 UD |
558 | aiocbp->aiocb.aio_buf, |
559 | aiocbp->aiocb.aio_nbytes, | |
560 | aiocbp->aiocb.aio_offset)); | |
561 | ||
562 | if (aiocbp->aiocb.__return_value == -1 && errno == ESPIPE) | |
563 | /* The Linux kernel is different from others. It returns | |
564 | ESPIPE if using pwrite on a socket. Other platforms | |
565 | simply ignore the offset parameter and behave like | |
566 | write. */ | |
567 | aiocbp->aiocb.__return_value = | |
568 | TEMP_FAILURE_RETRY (write (fildes, | |
d71b808a | 569 | (void *) aiocbp->aiocb64.aio_buf, |
a3bfd999 UD |
570 | aiocbp->aiocb64.aio_nbytes)); |
571 | } | |
572 | else if (aiocbp->aiocb.aio_lio_opcode == LIO_DSYNC) | |
d71b808a | 573 | aiocbp->aiocb.__return_value = |
a3bfd999 UD |
574 | TEMP_FAILURE_RETRY (fdatasync (fildes)); |
575 | else if (aiocbp->aiocb.aio_lio_opcode == LIO_SYNC) | |
754549b3 | 576 | aiocbp->aiocb.__return_value = |
a3bfd999 | 577 | TEMP_FAILURE_RETRY (fsync (fildes)); |
cbdee279 | 578 | else |
a3bfd999 UD |
579 | { |
580 | /* This is an invalid opcode. */ | |
581 | aiocbp->aiocb.__return_value = -1; | |
582 | __set_errno (EINVAL); | |
583 | } | |
cbdee279 | 584 | |
a3bfd999 UD |
585 | /* Get the mutex. */ |
586 | pthread_mutex_lock (&__aio_requests_mutex); | |
d71b808a | 587 | |
a3bfd999 UD |
588 | if (aiocbp->aiocb.__return_value == -1) |
589 | aiocbp->aiocb.__error_code = errno; | |
590 | else | |
591 | aiocbp->aiocb.__error_code = 0; | |
cbdee279 | 592 | |
a3bfd999 UD |
593 | /* Send the signal to notify about finished processing of the |
594 | request. */ | |
595 | __aio_notify (runp); | |
d71b808a | 596 | |
56ddf355 UD |
597 | /* For debugging purposes we reset the running flag of the |
598 | finished request. */ | |
599 | assert (runp->running == allocated); | |
600 | runp->running = done; | |
601 | ||
a3bfd999 | 602 | /* Now dequeue the current request. */ |
92806ee9 UD |
603 | __aio_remove_request (NULL, runp, 0); |
604 | if (runp->next_prio != NULL) | |
605 | add_request_to_runlist (runp->next_prio); | |
d71b808a | 606 | |
a3bfd999 UD |
607 | /* Free the old element. */ |
608 | __aio_free_request (runp); | |
609 | } | |
cbdee279 | 610 | |
b9337b6a | 611 | runp = runlist; |
a3bfd999 UD |
612 | |
613 | /* If the runlist is empty, then we sleep for a while, waiting for | |
614 | something to arrive in it. */ | |
615 | if (runp == NULL && optim.aio_idle_time >= 0) | |
d71b808a | 616 | { |
a3bfd999 UD |
617 | struct timeval now; |
618 | struct timespec wakeup_time; | |
619 | ||
620 | ++idle_thread_count; | |
c5c2b7c3 | 621 | __gettimeofday (&now, NULL); |
a3bfd999 UD |
622 | wakeup_time.tv_sec = now.tv_sec + optim.aio_idle_time; |
623 | wakeup_time.tv_nsec = now.tv_usec * 1000; | |
82d86f28 | 624 | if (wakeup_time.tv_nsec >= 1000000000) |
cbdee279 | 625 | { |
a3bfd999 UD |
626 | wakeup_time.tv_nsec -= 1000000000; |
627 | ++wakeup_time.tv_sec; | |
cbdee279 | 628 | } |
a3bfd999 UD |
629 | pthread_cond_timedwait (&__aio_new_request_notification, |
630 | &__aio_requests_mutex, | |
631 | &wakeup_time); | |
632 | --idle_thread_count; | |
633 | runp = runlist; | |
cbdee279 | 634 | } |
cbdee279 | 635 | |
d71b808a UD |
636 | if (runp == NULL) |
637 | --nthreads; | |
b9337b6a | 638 | else |
a3bfd999 UD |
639 | { |
640 | assert (runp->running == yes); | |
641 | runp->running = allocated; | |
642 | runlist = runp->next_run; | |
643 | ||
644 | /* If we have a request to process, and there's still another in | |
645 | the run list, then we need to either wake up or create a new | |
646 | thread to service the request that is still in the run list. */ | |
647 | if (runlist != NULL) | |
648 | { | |
649 | /* There are at least two items in the work queue to work on. | |
650 | If there are other idle threads, then we should wake them | |
651 | up for these other work elements; otherwise, we should try | |
652 | to create a new thread. */ | |
653 | if (idle_thread_count > 0) | |
654 | pthread_cond_signal (&__aio_new_request_notification); | |
655 | else if (nthreads < optim.aio_threads) | |
656 | { | |
657 | pthread_t thid; | |
658 | pthread_attr_t attr; | |
659 | ||
660 | /* Make sure the thread is created detached. */ | |
661 | pthread_attr_init (&attr); | |
662 | pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED); | |
663 | ||
664 | /* Now try to start a thread. If we fail, no big deal, | |
665 | because we know that there is at least one thread (us) | |
666 | that is working on AIO operations. */ | |
667 | if (pthread_create (&thid, &attr, handle_fildes_io, NULL) | |
668 | == 0) | |
669 | ++nthreads; | |
670 | } | |
671 | } | |
672 | } | |
cbdee279 | 673 | |
d71b808a UD |
674 | /* Release the mutex. */ |
675 | pthread_mutex_unlock (&__aio_requests_mutex); | |
cbdee279 UD |
676 | } |
677 | while (runp != NULL); | |
678 | ||
3cabdafa | 679 | return NULL; |
cbdee279 | 680 | } |
d71b808a UD |
681 | |
682 | ||
683 | /* Free allocated resources. */ | |
c877418f | 684 | libc_freeres_fn (free_res) |
d71b808a UD |
685 | { |
686 | size_t row; | |
687 | ||
56ddf355 | 688 | for (row = 0; row < pool_max_size; ++row) |
d71b808a UD |
689 | free (pool[row]); |
690 | ||
691 | free (pool); | |
692 | } | |
a3bfd999 UD |
693 | |
694 | ||
695 | /* Add newrequest to the runlist. The __abs_prio flag of newrequest must | |
696 | be correctly set to do this. Also, you had better set newrequest's | |
697 | "running" flag to "yes" before you release your lock or you'll throw an | |
698 | assertion. */ | |
699 | static void | |
700 | add_request_to_runlist (struct requestlist *newrequest) | |
701 | { | |
702 | int prio = newrequest->aiocbp->aiocb.__abs_prio; | |
703 | struct requestlist *runp; | |
704 | ||
705 | if (runlist == NULL || runlist->aiocbp->aiocb.__abs_prio < prio) | |
706 | { | |
707 | newrequest->next_run = runlist; | |
708 | runlist = newrequest; | |
709 | } | |
710 | else | |
711 | { | |
712 | runp = runlist; | |
713 | ||
714 | while (runp->next_run != NULL | |
715 | && runp->next_run->aiocbp->aiocb.__abs_prio >= prio) | |
716 | runp = runp->next_run; | |
717 | ||
718 | newrequest->next_run = runp->next_run; | |
719 | runp->next_run = newrequest; | |
720 | } | |
721 | } |