1 #include "git-compat-util.h"
4 #include "environment.h"
7 #include "fsmonitor-ipc.h"
8 #include "run-command.h"
12 #define INDEX_EXTENSION_VERSION1 (1)
13 #define INDEX_EXTENSION_VERSION2 (2)
14 #define HOOK_INTERFACE_VERSION1 (1)
15 #define HOOK_INTERFACE_VERSION2 (2)
17 struct trace_key trace_fsmonitor
= TRACE_KEY_INIT(FSMONITOR
);
19 static void assert_index_minimum(struct index_state
*istate
, size_t pos
)
21 if (pos
> istate
->cache_nr
)
22 BUG("fsmonitor_dirty has more entries than the index (%"PRIuMAX
" > %u)",
23 (uintmax_t)pos
, istate
->cache_nr
);
26 static void fsmonitor_ewah_callback(size_t pos
, void *is
)
28 struct index_state
*istate
= (struct index_state
*)is
;
29 struct cache_entry
*ce
;
31 assert_index_minimum(istate
, pos
+ 1);
33 ce
= istate
->cache
[pos
];
34 ce
->ce_flags
&= ~CE_FSMONITOR_VALID
;
37 static int fsmonitor_hook_version(void)
41 if (git_config_get_int("core.fsmonitorhookversion", &hook_version
))
44 if (hook_version
== HOOK_INTERFACE_VERSION1
||
45 hook_version
== HOOK_INTERFACE_VERSION2
)
48 warning("Invalid hook version '%i' in core.fsmonitorhookversion. "
49 "Must be 1 or 2.", hook_version
);
53 int read_fsmonitor_extension(struct index_state
*istate
, const void *data
,
56 const char *index
= data
;
59 struct ewah_bitmap
*fsmonitor_dirty
;
62 struct strbuf last_update
= STRBUF_INIT
;
64 if (sz
< sizeof(uint32_t) + 1 + sizeof(uint32_t))
65 return error("corrupt fsmonitor extension (too short)");
67 hdr_version
= get_be32(index
);
68 index
+= sizeof(uint32_t);
69 if (hdr_version
== INDEX_EXTENSION_VERSION1
) {
70 timestamp
= get_be64(index
);
71 strbuf_addf(&last_update
, "%"PRIu64
"", timestamp
);
72 index
+= sizeof(uint64_t);
73 } else if (hdr_version
== INDEX_EXTENSION_VERSION2
) {
74 strbuf_addstr(&last_update
, index
);
75 index
+= last_update
.len
+ 1;
77 return error("bad fsmonitor version %d", hdr_version
);
80 istate
->fsmonitor_last_update
= strbuf_detach(&last_update
, NULL
);
82 ewah_size
= get_be32(index
);
83 index
+= sizeof(uint32_t);
85 fsmonitor_dirty
= ewah_new();
86 ret
= ewah_read_mmap(fsmonitor_dirty
, index
, ewah_size
);
87 if (ret
!= ewah_size
) {
88 ewah_free(fsmonitor_dirty
);
89 return error("failed to parse ewah bitmap reading fsmonitor index extension");
91 istate
->fsmonitor_dirty
= fsmonitor_dirty
;
93 if (!istate
->split_index
)
94 assert_index_minimum(istate
, istate
->fsmonitor_dirty
->bit_size
);
96 trace2_data_string("index", NULL
, "extension/fsmn/read/token",
97 istate
->fsmonitor_last_update
);
98 trace_printf_key(&trace_fsmonitor
,
99 "read fsmonitor extension successful '%s'",
100 istate
->fsmonitor_last_update
);
104 void fill_fsmonitor_bitmap(struct index_state
*istate
)
106 unsigned int i
, skipped
= 0;
107 istate
->fsmonitor_dirty
= ewah_new();
108 for (i
= 0; i
< istate
->cache_nr
; i
++) {
109 if (istate
->cache
[i
]->ce_flags
& CE_REMOVE
)
111 else if (!(istate
->cache
[i
]->ce_flags
& CE_FSMONITOR_VALID
))
112 ewah_set(istate
->fsmonitor_dirty
, i
- skipped
);
116 void write_fsmonitor_extension(struct strbuf
*sb
, struct index_state
*istate
)
118 uint32_t hdr_version
;
120 uint32_t ewah_size
= 0;
123 if (!istate
->split_index
)
124 assert_index_minimum(istate
, istate
->fsmonitor_dirty
->bit_size
);
126 put_be32(&hdr_version
, INDEX_EXTENSION_VERSION2
);
127 strbuf_add(sb
, &hdr_version
, sizeof(uint32_t));
129 strbuf_addstr(sb
, istate
->fsmonitor_last_update
);
130 strbuf_addch(sb
, 0); /* Want to keep a NUL */
133 strbuf_add(sb
, &ewah_size
, sizeof(uint32_t)); /* we'll fix this up later */
135 ewah_start
= sb
->len
;
136 ewah_serialize_strbuf(istate
->fsmonitor_dirty
, sb
);
137 ewah_free(istate
->fsmonitor_dirty
);
138 istate
->fsmonitor_dirty
= NULL
;
140 /* fix up size field */
141 put_be32(&ewah_size
, sb
->len
- ewah_start
);
142 memcpy(sb
->buf
+ fixup
, &ewah_size
, sizeof(uint32_t));
144 trace2_data_string("index", NULL
, "extension/fsmn/write/token",
145 istate
->fsmonitor_last_update
);
146 trace_printf_key(&trace_fsmonitor
,
147 "write fsmonitor extension successful '%s'",
148 istate
->fsmonitor_last_update
);
152 * Call the query-fsmonitor hook passing the last update token of the saved results.
154 static int query_fsmonitor_hook(struct repository
*r
,
156 const char *last_update
,
157 struct strbuf
*query_result
)
159 struct child_process cp
= CHILD_PROCESS_INIT
;
162 if (fsm_settings__get_mode(r
) != FSMONITOR_MODE_HOOK
)
165 strvec_push(&cp
.args
, fsm_settings__get_hook_path(r
));
166 strvec_pushf(&cp
.args
, "%d", version
);
167 strvec_pushf(&cp
.args
, "%s", last_update
);
169 cp
.dir
= get_git_work_tree();
171 trace2_region_enter("fsm_hook", "query", NULL
);
173 result
= capture_command(&cp
, query_result
, 1024);
176 trace2_data_intmax("fsm_hook", NULL
, "query/failed", result
);
178 trace2_data_intmax("fsm_hook", NULL
, "query/response-length",
181 trace2_region_leave("fsm_hook", "query", NULL
);
186 static size_t handle_path_with_trailing_slash(
187 struct index_state
*istate
, const char *name
, int pos
);
190 * The daemon sent an observed pathname without a trailing slash.
191 * (This is the normal case.) We do not know if it is a tracked or
192 * untracked file, a sparse-directory, or a populated directory (on a
193 * platform such as Windows where FSEvents are not qualified).
195 * The pathname contains the observed case reported by the FS. We
196 * do not know it is case-correct or -incorrect.
198 * Assume it is case-correct and try an exact match.
200 static void handle_path_without_trailing_slash(
201 struct index_state
*istate
, const char *name
, int pos
)
204 * Mark the untracked cache dirty for this path (regardless of
205 * whether or not we find an exact match for it in the index).
206 * Since the path is unqualified (no trailing slash hint in the
207 * FSEvent), it may refer to a file or directory. So we should
208 * not assume one or the other and should always let the untracked
209 * cache decide what needs to invalidated.
211 untracked_cache_invalidate_trimmed_path(istate
, name
, 0);
215 * An exact match on a tracked file. We assume that we
216 * do not need to scan forward for a sparse-directory
217 * cache-entry with the same pathname, nor for a cone
218 * at that directory. (That is, assume no D/F conflicts.)
220 istate
->cache
[pos
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
222 struct strbuf work_path
= STRBUF_INIT
;
225 * The negative "pos" gives us the suggested insertion
226 * point for the pathname (without the trailing slash).
227 * We need to see if there is a directory with that
228 * prefix, but there can be lots of pathnames between
229 * "foo" and "foo/" like "foo-" or "foo-bar", so we
230 * don't want to do our own scan.
232 strbuf_add(&work_path
, name
, strlen(name
));
233 strbuf_addch(&work_path
, '/');
234 pos
= index_name_pos(istate
, work_path
.buf
, work_path
.len
);
235 handle_path_with_trailing_slash(istate
, work_path
.buf
, pos
);
236 strbuf_release(&work_path
);
241 * The daemon can decorate directory events, such as a move or rename,
242 * by adding a trailing slash to the observed name. Use this to
243 * explicitly invalidate the entire cone under that directory.
245 * The daemon can only reliably do that if the OS FSEvent contains
246 * sufficient information in the event.
248 * macOS FSEvents have enough information.
250 * Other platforms may or may not be able to do it (and it might
251 * depend on the type of event (for example, a daemon could lstat() an
252 * observed pathname after a rename, but not after a delete)).
254 * If we find an exact match in the index for a path with a trailing
255 * slash, it means that we matched a sparse-index directory in a
256 * cone-mode sparse-checkout (since that's the only time we have
257 * directories in the index). We should never see this in practice
258 * (because sparse directories should not be present and therefore
259 * not generating FS events). Either way, we can treat them in the
260 * same way and just invalidate the cache-entry and the untracked
261 * cache (and in this case, the forward cache-entry scan won't find
262 * anything and it doesn't hurt to let it run).
264 * Return the number of cache-entries that we invalidated. We will
265 * use this later to determine if we need to attempt a second
266 * case-insensitive search on case-insensitive file systems. That is,
267 * if the search using the observed-case in the FSEvent yields any
268 * results, we assume the prefix is case-correct. If there are no
269 * matches, we still don't know if the observed path is simply
270 * untracked or case-incorrect.
272 static size_t handle_path_with_trailing_slash(
273 struct index_state
*istate
, const char *name
, int pos
)
276 size_t nr_in_cone
= 0;
279 * Mark the untracked cache dirty for this directory path
280 * (regardless of whether or not we find an exact match for it
281 * in the index or find it to be proper prefix of one or more
282 * files in the index), since the FSEvent is hinting that
283 * there may be changes on or within the directory.
285 untracked_cache_invalidate_trimmed_path(istate
, name
, 0);
290 /* Mark all entries for the folder invalid */
291 for (i
= pos
; i
< istate
->cache_nr
; i
++) {
292 if (!starts_with(istate
->cache
[i
]->name
, name
))
294 istate
->cache
[i
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
301 static void fsmonitor_refresh_callback(struct index_state
*istate
, char *name
)
303 int len
= strlen(name
);
304 int pos
= index_name_pos(istate
, name
, len
);
306 trace_printf_key(&trace_fsmonitor
,
307 "fsmonitor_refresh_callback '%s' (pos %d)",
310 if (name
[len
- 1] == '/') {
311 handle_path_with_trailing_slash(istate
, name
, pos
);
313 handle_path_without_trailing_slash(istate
, name
, pos
);
318 * The number of pathnames that we need to receive from FSMonitor
319 * before we force the index to be updated.
321 * Note that any pathname within the set of received paths MAY cause
322 * cache-entry or istate flag bits to be updated and thus cause the
323 * index to be updated on disk.
325 * However, the response may contain many paths (such as ignored
326 * paths) that will not update any flag bits. And thus not force the
327 * index to be updated. (This is fine and normal.) It also means
328 * that the token will not be updated in the FSMonitor index
329 * extension. So the next Git command will find the same token in the
330 * index, make the same token-relative request, and receive the same
331 * response (plus any newly changed paths). If this response is large
332 * (and continues to grow), performance could be impacted.
334 * For example, if the user runs a build and it writes 100K object
335 * files but doesn't modify any source files, the index would not need
336 * to be updated. The FSMonitor response (after the build and
337 * relative to a pre-build token) might be 5MB. Each subsequent Git
338 * command will receive that same 100K/5MB response until something
339 * causes the index to be updated. And `refresh_fsmonitor()` will
340 * have to iterate over those 100K paths each time.
342 * Performance could be improved if we optionally force update the
343 * index after a very large response and get an updated token into
344 * the FSMonitor index extension. This should allow subsequent
345 * commands to get smaller and more current responses.
347 * The value chosen here does not need to be precise. The index
348 * will be updated automatically the first time the user touches
349 * a tracked file and causes a command like `git status` to
350 * update an mtime to be updated and/or set a flag bit.
352 static int fsmonitor_force_update_threshold
= 100;
354 void refresh_fsmonitor(struct index_state
*istate
)
356 static int warn_once
= 0;
357 struct strbuf query_result
= STRBUF_INIT
;
358 int query_success
= 0, hook_version
= -1;
359 size_t bol
= 0; /* beginning of line */
360 uint64_t last_update
;
361 struct strbuf last_update_token
= STRBUF_INIT
;
365 struct repository
*r
= istate
->repo
;
366 enum fsmonitor_mode fsm_mode
= fsm_settings__get_mode(r
);
367 enum fsmonitor_reason reason
= fsm_settings__get_reason(r
);
369 if (!warn_once
&& reason
> FSMONITOR_REASON_OK
) {
370 char *msg
= fsm_settings__get_incompatible_msg(r
, reason
);
376 if (fsm_mode
<= FSMONITOR_MODE_DISABLED
||
377 istate
->fsmonitor_has_run_once
)
380 istate
->fsmonitor_has_run_once
= 1;
382 trace_printf_key(&trace_fsmonitor
, "refresh fsmonitor");
384 if (fsm_mode
== FSMONITOR_MODE_IPC
) {
385 query_success
= !fsmonitor_ipc__send_query(
386 istate
->fsmonitor_last_update
?
387 istate
->fsmonitor_last_update
: "builtin:fake",
391 * The response contains a series of nul terminated
392 * strings. The first is the new token.
394 * Use `char *buf` as an interlude to trick the CI
395 * static analysis to let us use `strbuf_addstr()`
396 * here (and only copy the token) rather than
399 buf
= query_result
.buf
;
400 strbuf_addstr(&last_update_token
, buf
);
401 bol
= last_update_token
.len
+ 1;
402 is_trivial
= query_result
.buf
[bol
] == '/';
404 trace2_data_intmax("fsm_client", NULL
,
405 "query/trivial-response", 1);
408 * The builtin daemon is not available on this
409 * platform -OR- we failed to get a response.
411 * Generate a fake token (rather than a V1
412 * timestamp) for the index extension. (If
413 * they switch back to the hook API, we don't
414 * want ambiguous state.)
416 strbuf_addstr(&last_update_token
, "builtin:fake");
422 assert(fsm_mode
== FSMONITOR_MODE_HOOK
);
424 hook_version
= fsmonitor_hook_version();
427 * This could be racy so save the date/time now and query_fsmonitor_hook
428 * should be inclusive to ensure we don't miss potential changes.
430 last_update
= getnanotime();
431 if (hook_version
== HOOK_INTERFACE_VERSION1
)
432 strbuf_addf(&last_update_token
, "%"PRIu64
"", last_update
);
435 * If we have a last update token, call query_fsmonitor_hook for the set of
436 * changes since that token, else assume everything is possibly dirty
439 if (istate
->fsmonitor_last_update
) {
440 if (hook_version
== -1 || hook_version
== HOOK_INTERFACE_VERSION2
) {
441 query_success
= !query_fsmonitor_hook(
442 r
, HOOK_INTERFACE_VERSION2
,
443 istate
->fsmonitor_last_update
, &query_result
);
446 if (hook_version
< 0)
447 hook_version
= HOOK_INTERFACE_VERSION2
;
450 * First entry will be the last update token
451 * Need to use a char * variable because static
452 * analysis was suggesting to use strbuf_addbuf
453 * but we don't want to copy the entire strbuf
454 * only the chars up to the first NUL
456 buf
= query_result
.buf
;
457 strbuf_addstr(&last_update_token
, buf
);
458 if (!last_update_token
.len
) {
459 warning("Empty last update token.");
462 bol
= last_update_token
.len
+ 1;
463 is_trivial
= query_result
.buf
[bol
] == '/';
465 } else if (hook_version
< 0) {
466 hook_version
= HOOK_INTERFACE_VERSION1
;
467 if (!last_update_token
.len
)
468 strbuf_addf(&last_update_token
, "%"PRIu64
"", last_update
);
472 if (hook_version
== HOOK_INTERFACE_VERSION1
) {
473 query_success
= !query_fsmonitor_hook(
474 r
, HOOK_INTERFACE_VERSION1
,
475 istate
->fsmonitor_last_update
, &query_result
);
477 is_trivial
= query_result
.buf
[0] == '/';
481 trace2_data_intmax("fsm_hook", NULL
,
482 "query/trivial-response", 1);
484 trace_performance_since(last_update
, "fsmonitor process '%s'",
485 fsm_settings__get_hook_path(r
));
486 trace_printf_key(&trace_fsmonitor
,
487 "fsmonitor process '%s' returned %s",
488 fsm_settings__get_hook_path(r
),
489 query_success
? "success" : "failure");
494 * The response from FSMonitor (excluding the header token) is
497 * [a] a (possibly empty) list of NUL delimited relative
498 * pathnames of changed paths. This list can contain
499 * files and directories. Directories have a trailing
502 * [b] a single '/' to indicate the provider had no
503 * information and that we should consider everything
504 * invalid. We call this a trivial response.
506 trace2_region_enter("fsmonitor", "apply_results", istate
->repo
);
508 if (query_success
&& !is_trivial
) {
510 * Mark all pathnames returned by the monitor as dirty.
512 * This updates both the cache-entries and the untracked-cache.
516 buf
= query_result
.buf
;
517 for (i
= bol
; i
< query_result
.len
; i
++) {
520 fsmonitor_refresh_callback(istate
, buf
+ bol
);
524 if (bol
< query_result
.len
) {
525 fsmonitor_refresh_callback(istate
, buf
+ bol
);
529 /* Now mark the untracked cache for fsmonitor usage */
530 if (istate
->untracked
)
531 istate
->untracked
->use_fsmonitor
= 1;
533 if (count
> fsmonitor_force_update_threshold
)
534 istate
->cache_changed
|= FSMONITOR_CHANGED
;
536 trace2_data_intmax("fsmonitor", istate
->repo
, "apply_count",
541 * We failed to get a response or received a trivial response,
542 * so invalidate everything.
544 * We only want to run the post index changed hook if
545 * we've actually changed entries, so keep track if we
546 * actually changed entries or not.
548 int is_cache_changed
= 0;
550 for (i
= 0; i
< istate
->cache_nr
; i
++) {
551 if (istate
->cache
[i
]->ce_flags
& CE_FSMONITOR_VALID
) {
552 is_cache_changed
= 1;
553 istate
->cache
[i
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
558 * If we're going to check every file, ensure we save
561 if (is_cache_changed
)
562 istate
->cache_changed
|= FSMONITOR_CHANGED
;
564 if (istate
->untracked
)
565 istate
->untracked
->use_fsmonitor
= 0;
567 trace2_region_leave("fsmonitor", "apply_results", istate
->repo
);
569 strbuf_release(&query_result
);
571 /* Now that we've updated istate, save the last_update_token */
572 FREE_AND_NULL(istate
->fsmonitor_last_update
);
573 istate
->fsmonitor_last_update
= strbuf_detach(&last_update_token
, NULL
);
577 * The caller wants to turn on FSMonitor. And when the caller writes
578 * the index to disk, a FSMonitor extension should be included. This
579 * requires that `istate->fsmonitor_last_update` not be NULL. But we
580 * have not actually talked to a FSMonitor process yet, so we don't
581 * have an initial value for this field.
583 * For a protocol V1 FSMonitor process, this field is a formatted
584 * "nanoseconds since epoch" field. However, for a protocol V2
585 * FSMonitor process, this field is an opaque token.
587 * Historically, `add_fsmonitor()` has initialized this field to the
588 * current time for protocol V1 processes. There are lots of race
589 * conditions here, but that code has shipped...
591 * The only true solution is to use a V2 FSMonitor and get a current
592 * or default token value (that it understands), but we cannot do that
593 * until we have actually talked to an instance of the FSMonitor process
594 * (but the protocol requires that we send a token first...).
596 * For simplicity, just initialize like we have a V1 process and require
597 * that V2 processes adapt.
599 static void initialize_fsmonitor_last_update(struct index_state
*istate
)
601 struct strbuf last_update
= STRBUF_INIT
;
603 strbuf_addf(&last_update
, "%"PRIu64
"", getnanotime());
604 istate
->fsmonitor_last_update
= strbuf_detach(&last_update
, NULL
);
607 void add_fsmonitor(struct index_state
*istate
)
611 if (!istate
->fsmonitor_last_update
) {
612 trace_printf_key(&trace_fsmonitor
, "add fsmonitor");
613 istate
->cache_changed
|= FSMONITOR_CHANGED
;
614 initialize_fsmonitor_last_update(istate
);
616 /* reset the fsmonitor state */
617 for (i
= 0; i
< istate
->cache_nr
; i
++)
618 istate
->cache
[i
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
620 /* reset the untracked cache */
621 if (istate
->untracked
) {
622 add_untracked_cache(istate
);
623 istate
->untracked
->use_fsmonitor
= 1;
626 /* Update the fsmonitor state */
627 refresh_fsmonitor(istate
);
631 void remove_fsmonitor(struct index_state
*istate
)
633 if (istate
->fsmonitor_last_update
) {
634 trace_printf_key(&trace_fsmonitor
, "remove fsmonitor");
635 istate
->cache_changed
|= FSMONITOR_CHANGED
;
636 FREE_AND_NULL(istate
->fsmonitor_last_update
);
640 void tweak_fsmonitor(struct index_state
*istate
)
643 int fsmonitor_enabled
= (fsm_settings__get_mode(istate
->repo
)
644 > FSMONITOR_MODE_DISABLED
);
646 if (istate
->fsmonitor_dirty
) {
647 if (fsmonitor_enabled
) {
648 /* Mark all entries valid */
649 for (i
= 0; i
< istate
->cache_nr
; i
++) {
650 if (S_ISGITLINK(istate
->cache
[i
]->ce_mode
))
652 istate
->cache
[i
]->ce_flags
|= CE_FSMONITOR_VALID
;
655 /* Mark all previously saved entries as dirty */
656 assert_index_minimum(istate
, istate
->fsmonitor_dirty
->bit_size
);
657 ewah_each_bit(istate
->fsmonitor_dirty
, fsmonitor_ewah_callback
, istate
);
659 refresh_fsmonitor(istate
);
662 ewah_free(istate
->fsmonitor_dirty
);
663 istate
->fsmonitor_dirty
= NULL
;
666 if (fsmonitor_enabled
)
667 add_fsmonitor(istate
);
669 remove_fsmonitor(istate
);