1 #include "git-compat-util.h"
4 #include "environment.h"
7 #include "fsmonitor-ipc.h"
8 #include "run-command.h"
12 #define INDEX_EXTENSION_VERSION1 (1)
13 #define INDEX_EXTENSION_VERSION2 (2)
14 #define HOOK_INTERFACE_VERSION1 (1)
15 #define HOOK_INTERFACE_VERSION2 (2)
17 struct trace_key trace_fsmonitor
= TRACE_KEY_INIT(FSMONITOR
);
19 static void assert_index_minimum(struct index_state
*istate
, size_t pos
)
21 if (pos
> istate
->cache_nr
)
22 BUG("fsmonitor_dirty has more entries than the index (%"PRIuMAX
" > %u)",
23 (uintmax_t)pos
, istate
->cache_nr
);
26 static void fsmonitor_ewah_callback(size_t pos
, void *is
)
28 struct index_state
*istate
= (struct index_state
*)is
;
29 struct cache_entry
*ce
;
31 assert_index_minimum(istate
, pos
+ 1);
33 ce
= istate
->cache
[pos
];
34 ce
->ce_flags
&= ~CE_FSMONITOR_VALID
;
37 static int fsmonitor_hook_version(void)
41 if (git_config_get_int("core.fsmonitorhookversion", &hook_version
))
44 if (hook_version
== HOOK_INTERFACE_VERSION1
||
45 hook_version
== HOOK_INTERFACE_VERSION2
)
48 warning("Invalid hook version '%i' in core.fsmonitorhookversion. "
49 "Must be 1 or 2.", hook_version
);
53 int read_fsmonitor_extension(struct index_state
*istate
, const void *data
,
56 const char *index
= data
;
59 struct ewah_bitmap
*fsmonitor_dirty
;
62 struct strbuf last_update
= STRBUF_INIT
;
64 if (sz
< sizeof(uint32_t) + 1 + sizeof(uint32_t))
65 return error("corrupt fsmonitor extension (too short)");
67 hdr_version
= get_be32(index
);
68 index
+= sizeof(uint32_t);
69 if (hdr_version
== INDEX_EXTENSION_VERSION1
) {
70 timestamp
= get_be64(index
);
71 strbuf_addf(&last_update
, "%"PRIu64
"", timestamp
);
72 index
+= sizeof(uint64_t);
73 } else if (hdr_version
== INDEX_EXTENSION_VERSION2
) {
74 strbuf_addstr(&last_update
, index
);
75 index
+= last_update
.len
+ 1;
77 return error("bad fsmonitor version %d", hdr_version
);
80 istate
->fsmonitor_last_update
= strbuf_detach(&last_update
, NULL
);
82 ewah_size
= get_be32(index
);
83 index
+= sizeof(uint32_t);
85 fsmonitor_dirty
= ewah_new();
86 ret
= ewah_read_mmap(fsmonitor_dirty
, index
, ewah_size
);
87 if (ret
!= ewah_size
) {
88 ewah_free(fsmonitor_dirty
);
89 return error("failed to parse ewah bitmap reading fsmonitor index extension");
91 istate
->fsmonitor_dirty
= fsmonitor_dirty
;
93 if (!istate
->split_index
)
94 assert_index_minimum(istate
, istate
->fsmonitor_dirty
->bit_size
);
96 trace2_data_string("index", NULL
, "extension/fsmn/read/token",
97 istate
->fsmonitor_last_update
);
98 trace_printf_key(&trace_fsmonitor
,
99 "read fsmonitor extension successful '%s'",
100 istate
->fsmonitor_last_update
);
104 void fill_fsmonitor_bitmap(struct index_state
*istate
)
106 unsigned int i
, skipped
= 0;
107 istate
->fsmonitor_dirty
= ewah_new();
108 for (i
= 0; i
< istate
->cache_nr
; i
++) {
109 if (istate
->cache
[i
]->ce_flags
& CE_REMOVE
)
111 else if (!(istate
->cache
[i
]->ce_flags
& CE_FSMONITOR_VALID
))
112 ewah_set(istate
->fsmonitor_dirty
, i
- skipped
);
116 void write_fsmonitor_extension(struct strbuf
*sb
, struct index_state
*istate
)
118 uint32_t hdr_version
;
120 uint32_t ewah_size
= 0;
123 if (!istate
->split_index
)
124 assert_index_minimum(istate
, istate
->fsmonitor_dirty
->bit_size
);
126 put_be32(&hdr_version
, INDEX_EXTENSION_VERSION2
);
127 strbuf_add(sb
, &hdr_version
, sizeof(uint32_t));
129 strbuf_addstr(sb
, istate
->fsmonitor_last_update
);
130 strbuf_addch(sb
, 0); /* Want to keep a NUL */
133 strbuf_add(sb
, &ewah_size
, sizeof(uint32_t)); /* we'll fix this up later */
135 ewah_start
= sb
->len
;
136 ewah_serialize_strbuf(istate
->fsmonitor_dirty
, sb
);
137 ewah_free(istate
->fsmonitor_dirty
);
138 istate
->fsmonitor_dirty
= NULL
;
140 /* fix up size field */
141 put_be32(&ewah_size
, sb
->len
- ewah_start
);
142 memcpy(sb
->buf
+ fixup
, &ewah_size
, sizeof(uint32_t));
144 trace2_data_string("index", NULL
, "extension/fsmn/write/token",
145 istate
->fsmonitor_last_update
);
146 trace_printf_key(&trace_fsmonitor
,
147 "write fsmonitor extension successful '%s'",
148 istate
->fsmonitor_last_update
);
152 * Call the query-fsmonitor hook passing the last update token of the saved results.
154 static int query_fsmonitor_hook(struct repository
*r
,
156 const char *last_update
,
157 struct strbuf
*query_result
)
159 struct child_process cp
= CHILD_PROCESS_INIT
;
162 if (fsm_settings__get_mode(r
) != FSMONITOR_MODE_HOOK
)
165 strvec_push(&cp
.args
, fsm_settings__get_hook_path(r
));
166 strvec_pushf(&cp
.args
, "%d", version
);
167 strvec_pushf(&cp
.args
, "%s", last_update
);
169 cp
.dir
= get_git_work_tree();
171 trace2_region_enter("fsm_hook", "query", NULL
);
173 result
= capture_command(&cp
, query_result
, 1024);
176 trace2_data_intmax("fsm_hook", NULL
, "query/failed", result
);
178 trace2_data_intmax("fsm_hook", NULL
, "query/response-length",
181 trace2_region_leave("fsm_hook", "query", NULL
);
186 static void handle_path_without_trailing_slash(
187 struct index_state
*istate
, const char *name
, int pos
)
192 * Mark the untracked cache dirty for this path (regardless of
193 * whether or not we find an exact match for it in the index).
194 * Since the path is unqualified (no trailing slash hint in the
195 * FSEvent), it may refer to a file or directory. So we should
196 * not assume one or the other and should always let the untracked
197 * cache decide what needs to invalidated.
199 untracked_cache_invalidate_trimmed_path(istate
, name
, 0);
203 * We have an exact match for this path and can just
206 istate
->cache
[pos
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
209 * The path is not a tracked file -or- it is a
210 * directory event on a platform that cannot
211 * distinguish between file and directory events in
212 * the event handler, such as Windows.
214 * Scan as if it is a directory and invalidate the
215 * cone under it. (But remember to ignore items
216 * between "name" and "name/", such as "name-" and
219 int len
= strlen(name
);
222 for (i
= pos
; i
< istate
->cache_nr
; i
++) {
223 if (!starts_with(istate
->cache
[i
]->name
, name
))
225 if ((unsigned char)istate
->cache
[i
]->name
[len
] > '/')
227 if (istate
->cache
[i
]->name
[len
] == '/')
228 istate
->cache
[i
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
234 * The daemon can decorate directory events, such as a move or rename,
235 * by adding a trailing slash to the observed name. Use this to
236 * explicitly invalidate the entire cone under that directory.
238 * The daemon can only reliably do that if the OS FSEvent contains
239 * sufficient information in the event.
241 * macOS FSEvents have enough information.
243 * Other platforms may or may not be able to do it (and it might
244 * depend on the type of event (for example, a daemon could lstat() an
245 * observed pathname after a rename, but not after a delete)).
247 * If we find an exact match in the index for a path with a trailing
248 * slash, it means that we matched a sparse-index directory in a
249 * cone-mode sparse-checkout (since that's the only time we have
250 * directories in the index). We should never see this in practice
251 * (because sparse directories should not be present and therefore
252 * not generating FS events). Either way, we can treat them in the
253 * same way and just invalidate the cache-entry and the untracked
254 * cache (and in this case, the forward cache-entry scan won't find
255 * anything and it doesn't hurt to let it run).
257 * Return the number of cache-entries that we invalidated. We will
258 * use this later to determine if we need to attempt a second
259 * case-insensitive search on case-insensitive file systems. That is,
260 * if the search using the observed-case in the FSEvent yields any
261 * results, we assume the prefix is case-correct. If there are no
262 * matches, we still don't know if the observed path is simply
263 * untracked or case-incorrect.
265 static size_t handle_path_with_trailing_slash(
266 struct index_state
*istate
, const char *name
, int pos
)
269 size_t nr_in_cone
= 0;
272 * Mark the untracked cache dirty for this directory path
273 * (regardless of whether or not we find an exact match for it
274 * in the index or find it to be proper prefix of one or more
275 * files in the index), since the FSEvent is hinting that
276 * there may be changes on or within the directory.
278 untracked_cache_invalidate_trimmed_path(istate
, name
, 0);
283 /* Mark all entries for the folder invalid */
284 for (i
= pos
; i
< istate
->cache_nr
; i
++) {
285 if (!starts_with(istate
->cache
[i
]->name
, name
))
287 istate
->cache
[i
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
294 static void fsmonitor_refresh_callback(struct index_state
*istate
, char *name
)
296 int len
= strlen(name
);
297 int pos
= index_name_pos(istate
, name
, len
);
299 trace_printf_key(&trace_fsmonitor
,
300 "fsmonitor_refresh_callback '%s' (pos %d)",
303 if (name
[len
- 1] == '/') {
304 handle_path_with_trailing_slash(istate
, name
, pos
);
306 handle_path_without_trailing_slash(istate
, name
, pos
);
311 * The number of pathnames that we need to receive from FSMonitor
312 * before we force the index to be updated.
314 * Note that any pathname within the set of received paths MAY cause
315 * cache-entry or istate flag bits to be updated and thus cause the
316 * index to be updated on disk.
318 * However, the response may contain many paths (such as ignored
319 * paths) that will not update any flag bits. And thus not force the
320 * index to be updated. (This is fine and normal.) It also means
321 * that the token will not be updated in the FSMonitor index
322 * extension. So the next Git command will find the same token in the
323 * index, make the same token-relative request, and receive the same
324 * response (plus any newly changed paths). If this response is large
325 * (and continues to grow), performance could be impacted.
327 * For example, if the user runs a build and it writes 100K object
328 * files but doesn't modify any source files, the index would not need
329 * to be updated. The FSMonitor response (after the build and
330 * relative to a pre-build token) might be 5MB. Each subsequent Git
331 * command will receive that same 100K/5MB response until something
332 * causes the index to be updated. And `refresh_fsmonitor()` will
333 * have to iterate over those 100K paths each time.
335 * Performance could be improved if we optionally force update the
336 * index after a very large response and get an updated token into
337 * the FSMonitor index extension. This should allow subsequent
338 * commands to get smaller and more current responses.
340 * The value chosen here does not need to be precise. The index
341 * will be updated automatically the first time the user touches
342 * a tracked file and causes a command like `git status` to
343 * update an mtime to be updated and/or set a flag bit.
345 static int fsmonitor_force_update_threshold
= 100;
347 void refresh_fsmonitor(struct index_state
*istate
)
349 static int warn_once
= 0;
350 struct strbuf query_result
= STRBUF_INIT
;
351 int query_success
= 0, hook_version
= -1;
352 size_t bol
= 0; /* beginning of line */
353 uint64_t last_update
;
354 struct strbuf last_update_token
= STRBUF_INIT
;
358 struct repository
*r
= istate
->repo
;
359 enum fsmonitor_mode fsm_mode
= fsm_settings__get_mode(r
);
360 enum fsmonitor_reason reason
= fsm_settings__get_reason(r
);
362 if (!warn_once
&& reason
> FSMONITOR_REASON_OK
) {
363 char *msg
= fsm_settings__get_incompatible_msg(r
, reason
);
369 if (fsm_mode
<= FSMONITOR_MODE_DISABLED
||
370 istate
->fsmonitor_has_run_once
)
373 istate
->fsmonitor_has_run_once
= 1;
375 trace_printf_key(&trace_fsmonitor
, "refresh fsmonitor");
377 if (fsm_mode
== FSMONITOR_MODE_IPC
) {
378 query_success
= !fsmonitor_ipc__send_query(
379 istate
->fsmonitor_last_update
?
380 istate
->fsmonitor_last_update
: "builtin:fake",
384 * The response contains a series of nul terminated
385 * strings. The first is the new token.
387 * Use `char *buf` as an interlude to trick the CI
388 * static analysis to let us use `strbuf_addstr()`
389 * here (and only copy the token) rather than
392 buf
= query_result
.buf
;
393 strbuf_addstr(&last_update_token
, buf
);
394 bol
= last_update_token
.len
+ 1;
395 is_trivial
= query_result
.buf
[bol
] == '/';
397 trace2_data_intmax("fsm_client", NULL
,
398 "query/trivial-response", 1);
401 * The builtin daemon is not available on this
402 * platform -OR- we failed to get a response.
404 * Generate a fake token (rather than a V1
405 * timestamp) for the index extension. (If
406 * they switch back to the hook API, we don't
407 * want ambiguous state.)
409 strbuf_addstr(&last_update_token
, "builtin:fake");
415 assert(fsm_mode
== FSMONITOR_MODE_HOOK
);
417 hook_version
= fsmonitor_hook_version();
420 * This could be racy so save the date/time now and query_fsmonitor_hook
421 * should be inclusive to ensure we don't miss potential changes.
423 last_update
= getnanotime();
424 if (hook_version
== HOOK_INTERFACE_VERSION1
)
425 strbuf_addf(&last_update_token
, "%"PRIu64
"", last_update
);
428 * If we have a last update token, call query_fsmonitor_hook for the set of
429 * changes since that token, else assume everything is possibly dirty
432 if (istate
->fsmonitor_last_update
) {
433 if (hook_version
== -1 || hook_version
== HOOK_INTERFACE_VERSION2
) {
434 query_success
= !query_fsmonitor_hook(
435 r
, HOOK_INTERFACE_VERSION2
,
436 istate
->fsmonitor_last_update
, &query_result
);
439 if (hook_version
< 0)
440 hook_version
= HOOK_INTERFACE_VERSION2
;
443 * First entry will be the last update token
444 * Need to use a char * variable because static
445 * analysis was suggesting to use strbuf_addbuf
446 * but we don't want to copy the entire strbuf
447 * only the chars up to the first NUL
449 buf
= query_result
.buf
;
450 strbuf_addstr(&last_update_token
, buf
);
451 if (!last_update_token
.len
) {
452 warning("Empty last update token.");
455 bol
= last_update_token
.len
+ 1;
456 is_trivial
= query_result
.buf
[bol
] == '/';
458 } else if (hook_version
< 0) {
459 hook_version
= HOOK_INTERFACE_VERSION1
;
460 if (!last_update_token
.len
)
461 strbuf_addf(&last_update_token
, "%"PRIu64
"", last_update
);
465 if (hook_version
== HOOK_INTERFACE_VERSION1
) {
466 query_success
= !query_fsmonitor_hook(
467 r
, HOOK_INTERFACE_VERSION1
,
468 istate
->fsmonitor_last_update
, &query_result
);
470 is_trivial
= query_result
.buf
[0] == '/';
474 trace2_data_intmax("fsm_hook", NULL
,
475 "query/trivial-response", 1);
477 trace_performance_since(last_update
, "fsmonitor process '%s'",
478 fsm_settings__get_hook_path(r
));
479 trace_printf_key(&trace_fsmonitor
,
480 "fsmonitor process '%s' returned %s",
481 fsm_settings__get_hook_path(r
),
482 query_success
? "success" : "failure");
487 * The response from FSMonitor (excluding the header token) is
490 * [a] a (possibly empty) list of NUL delimited relative
491 * pathnames of changed paths. This list can contain
492 * files and directories. Directories have a trailing
495 * [b] a single '/' to indicate the provider had no
496 * information and that we should consider everything
497 * invalid. We call this a trivial response.
499 trace2_region_enter("fsmonitor", "apply_results", istate
->repo
);
501 if (query_success
&& !is_trivial
) {
503 * Mark all pathnames returned by the monitor as dirty.
505 * This updates both the cache-entries and the untracked-cache.
509 buf
= query_result
.buf
;
510 for (i
= bol
; i
< query_result
.len
; i
++) {
513 fsmonitor_refresh_callback(istate
, buf
+ bol
);
517 if (bol
< query_result
.len
) {
518 fsmonitor_refresh_callback(istate
, buf
+ bol
);
522 /* Now mark the untracked cache for fsmonitor usage */
523 if (istate
->untracked
)
524 istate
->untracked
->use_fsmonitor
= 1;
526 if (count
> fsmonitor_force_update_threshold
)
527 istate
->cache_changed
|= FSMONITOR_CHANGED
;
529 trace2_data_intmax("fsmonitor", istate
->repo
, "apply_count",
534 * We failed to get a response or received a trivial response,
535 * so invalidate everything.
537 * We only want to run the post index changed hook if
538 * we've actually changed entries, so keep track if we
539 * actually changed entries or not.
541 int is_cache_changed
= 0;
543 for (i
= 0; i
< istate
->cache_nr
; i
++) {
544 if (istate
->cache
[i
]->ce_flags
& CE_FSMONITOR_VALID
) {
545 is_cache_changed
= 1;
546 istate
->cache
[i
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
551 * If we're going to check every file, ensure we save
554 if (is_cache_changed
)
555 istate
->cache_changed
|= FSMONITOR_CHANGED
;
557 if (istate
->untracked
)
558 istate
->untracked
->use_fsmonitor
= 0;
560 trace2_region_leave("fsmonitor", "apply_results", istate
->repo
);
562 strbuf_release(&query_result
);
564 /* Now that we've updated istate, save the last_update_token */
565 FREE_AND_NULL(istate
->fsmonitor_last_update
);
566 istate
->fsmonitor_last_update
= strbuf_detach(&last_update_token
, NULL
);
570 * The caller wants to turn on FSMonitor. And when the caller writes
571 * the index to disk, a FSMonitor extension should be included. This
572 * requires that `istate->fsmonitor_last_update` not be NULL. But we
573 * have not actually talked to a FSMonitor process yet, so we don't
574 * have an initial value for this field.
576 * For a protocol V1 FSMonitor process, this field is a formatted
577 * "nanoseconds since epoch" field. However, for a protocol V2
578 * FSMonitor process, this field is an opaque token.
580 * Historically, `add_fsmonitor()` has initialized this field to the
581 * current time for protocol V1 processes. There are lots of race
582 * conditions here, but that code has shipped...
584 * The only true solution is to use a V2 FSMonitor and get a current
585 * or default token value (that it understands), but we cannot do that
586 * until we have actually talked to an instance of the FSMonitor process
587 * (but the protocol requires that we send a token first...).
589 * For simplicity, just initialize like we have a V1 process and require
590 * that V2 processes adapt.
592 static void initialize_fsmonitor_last_update(struct index_state
*istate
)
594 struct strbuf last_update
= STRBUF_INIT
;
596 strbuf_addf(&last_update
, "%"PRIu64
"", getnanotime());
597 istate
->fsmonitor_last_update
= strbuf_detach(&last_update
, NULL
);
600 void add_fsmonitor(struct index_state
*istate
)
604 if (!istate
->fsmonitor_last_update
) {
605 trace_printf_key(&trace_fsmonitor
, "add fsmonitor");
606 istate
->cache_changed
|= FSMONITOR_CHANGED
;
607 initialize_fsmonitor_last_update(istate
);
609 /* reset the fsmonitor state */
610 for (i
= 0; i
< istate
->cache_nr
; i
++)
611 istate
->cache
[i
]->ce_flags
&= ~CE_FSMONITOR_VALID
;
613 /* reset the untracked cache */
614 if (istate
->untracked
) {
615 add_untracked_cache(istate
);
616 istate
->untracked
->use_fsmonitor
= 1;
619 /* Update the fsmonitor state */
620 refresh_fsmonitor(istate
);
624 void remove_fsmonitor(struct index_state
*istate
)
626 if (istate
->fsmonitor_last_update
) {
627 trace_printf_key(&trace_fsmonitor
, "remove fsmonitor");
628 istate
->cache_changed
|= FSMONITOR_CHANGED
;
629 FREE_AND_NULL(istate
->fsmonitor_last_update
);
633 void tweak_fsmonitor(struct index_state
*istate
)
636 int fsmonitor_enabled
= (fsm_settings__get_mode(istate
->repo
)
637 > FSMONITOR_MODE_DISABLED
);
639 if (istate
->fsmonitor_dirty
) {
640 if (fsmonitor_enabled
) {
641 /* Mark all entries valid */
642 for (i
= 0; i
< istate
->cache_nr
; i
++) {
643 if (S_ISGITLINK(istate
->cache
[i
]->ce_mode
))
645 istate
->cache
[i
]->ce_flags
|= CE_FSMONITOR_VALID
;
648 /* Mark all previously saved entries as dirty */
649 assert_index_minimum(istate
, istate
->fsmonitor_dirty
->bit_size
);
650 ewah_each_bit(istate
->fsmonitor_dirty
, fsmonitor_ewah_callback
, istate
);
652 refresh_fsmonitor(istate
);
655 ewah_free(istate
->fsmonitor_dirty
);
656 istate
->fsmonitor_dirty
= NULL
;
659 if (fsmonitor_enabled
)
660 add_fsmonitor(istate
);
662 remove_fsmonitor(istate
);