1 #define NO_THE_INDEX_COMPATIBILITY_MACROS
5 #include "run-command.h"
9 #include "sub-process.h"
12 * convert.c - convert a file when checking it out and checking it in.
14 * This should use the pathname to decide on whether it wants to do some
15 * more interesting conversions (automatic gzip/unzip, general format
16 * conversions etc etc), but by default it just does automatic CRLF<->LF
17 * translation when the "text" attribute or "auto_crlf" option is set.
20 /* Stat bits: When BIN is set, the txt bits are unset */
21 #define CONVERT_STAT_BITS_TXT_LF 0x1
22 #define CONVERT_STAT_BITS_TXT_CRLF 0x2
23 #define CONVERT_STAT_BITS_BIN 0x4
37 /* NUL, CR, LF and CRLF counts */
38 unsigned nul
, lonecr
, lonelf
, crlf
;
40 /* These are just approximations! */
41 unsigned printable
, nonprintable
;
44 static void gather_stats(const char *buf
, unsigned long size
, struct text_stat
*stats
)
48 memset(stats
, 0, sizeof(*stats
));
50 for (i
= 0; i
< size
; i
++) {
51 unsigned char c
= buf
[i
];
53 if (i
+1 < size
&& buf
[i
+1] == '\n') {
66 stats
->nonprintable
++;
69 /* BS, HT, ESC and FF */
70 case '\b': case '\t': case '\033': case '\014':
77 stats
->nonprintable
++;
84 /* If file ends with EOF then don't count this EOF as non-printable. */
85 if (size
>= 1 && buf
[size
-1] == '\032')
86 stats
->nonprintable
--;
90 * The same heuristics as diff.c::mmfile_is_binary()
91 * We treat files with bare CR as binary
93 static int convert_is_binary(unsigned long size
, const struct text_stat
*stats
)
99 if ((stats
->printable
>> 7) < stats
->nonprintable
)
104 static unsigned int gather_convert_stats(const char *data
, unsigned long size
)
106 struct text_stat stats
;
110 gather_stats(data
, size
, &stats
);
111 if (convert_is_binary(size
, &stats
))
112 ret
|= CONVERT_STAT_BITS_BIN
;
114 ret
|= CONVERT_STAT_BITS_TXT_CRLF
;
116 ret
|= CONVERT_STAT_BITS_TXT_LF
;
121 static const char *gather_convert_stats_ascii(const char *data
, unsigned long size
)
123 unsigned int convert_stats
= gather_convert_stats(data
, size
);
125 if (convert_stats
& CONVERT_STAT_BITS_BIN
)
127 switch (convert_stats
) {
128 case CONVERT_STAT_BITS_TXT_LF
:
130 case CONVERT_STAT_BITS_TXT_CRLF
:
132 case CONVERT_STAT_BITS_TXT_LF
| CONVERT_STAT_BITS_TXT_CRLF
:
139 const char *get_cached_convert_stats_ascii(const struct index_state
*istate
,
144 void *data
= read_blob_data_from_index(istate
, path
, &sz
);
145 ret
= gather_convert_stats_ascii(data
, sz
);
150 const char *get_wt_convert_stats_ascii(const char *path
)
152 const char *ret
= "";
153 struct strbuf sb
= STRBUF_INIT
;
154 if (strbuf_read_file(&sb
, path
, 0) >= 0)
155 ret
= gather_convert_stats_ascii(sb
.buf
, sb
.len
);
160 static int text_eol_is_crlf(void)
162 if (auto_crlf
== AUTO_CRLF_TRUE
)
164 else if (auto_crlf
== AUTO_CRLF_INPUT
)
166 if (core_eol
== EOL_CRLF
)
168 if (core_eol
== EOL_UNSET
&& EOL_NATIVE
== EOL_CRLF
)
173 static enum eol
output_eol(enum crlf_action crlf_action
)
175 switch (crlf_action
) {
180 case CRLF_TEXT_INPUT
:
185 case CRLF_AUTO_INPUT
:
190 return text_eol_is_crlf() ? EOL_CRLF
: EOL_LF
;
192 warning("Illegal crlf_action %d\n", (int)crlf_action
);
196 static void check_safe_crlf(const char *path
, enum crlf_action crlf_action
,
197 struct text_stat
*old_stats
, struct text_stat
*new_stats
,
198 enum safe_crlf checksafe
)
200 if (old_stats
->crlf
&& !new_stats
->crlf
) {
202 * CRLFs would not be restored by checkout
204 if (checksafe
== SAFE_CRLF_WARN
)
205 warning(_("CRLF will be replaced by LF in %s.\n"
206 "The file will have its original line"
207 " endings in your working directory."), path
);
208 else /* i.e. SAFE_CRLF_FAIL */
209 die(_("CRLF would be replaced by LF in %s."), path
);
210 } else if (old_stats
->lonelf
&& !new_stats
->lonelf
) {
212 * CRLFs would be added by checkout
214 if (checksafe
== SAFE_CRLF_WARN
)
215 warning(_("LF will be replaced by CRLF in %s.\n"
216 "The file will have its original line"
217 " endings in your working directory."), path
);
218 else /* i.e. SAFE_CRLF_FAIL */
219 die(_("LF would be replaced by CRLF in %s"), path
);
223 static int has_cr_in_index(const struct index_state
*istate
, const char *path
)
229 data
= read_blob_data_from_index(istate
, path
, &sz
);
232 has_cr
= memchr(data
, '\r', sz
) != NULL
;
237 static int will_convert_lf_to_crlf(size_t len
, struct text_stat
*stats
,
238 enum crlf_action crlf_action
)
240 if (output_eol(crlf_action
) != EOL_CRLF
)
242 /* No "naked" LF? Nothing to convert, regardless. */
246 if (crlf_action
== CRLF_AUTO
|| crlf_action
== CRLF_AUTO_INPUT
|| crlf_action
== CRLF_AUTO_CRLF
) {
247 /* If we have any CR or CRLF line endings, we do not touch it */
248 /* This is the new safer autocrlf-handling */
249 if (stats
->lonecr
|| stats
->crlf
)
252 if (convert_is_binary(len
, stats
))
259 static int crlf_to_git(const struct index_state
*istate
,
260 const char *path
, const char *src
, size_t len
,
262 enum crlf_action crlf_action
, enum safe_crlf checksafe
)
264 struct text_stat stats
;
266 int convert_crlf_into_lf
;
268 if (crlf_action
== CRLF_BINARY
||
273 * If we are doing a dry-run and have no source buffer, there is
274 * nothing to analyze; we must assume we would convert.
279 gather_stats(src
, len
, &stats
);
280 /* Optimization: No CRLF? Nothing to convert, regardless. */
281 convert_crlf_into_lf
= !!stats
.crlf
;
283 if (crlf_action
== CRLF_AUTO
|| crlf_action
== CRLF_AUTO_INPUT
|| crlf_action
== CRLF_AUTO_CRLF
) {
284 if (convert_is_binary(len
, &stats
))
287 * If the file in the index has any CR in it, do not
288 * convert. This is the new safer autocrlf handling,
289 * unless we want to renormalize in a merge or
292 if ((checksafe
!= SAFE_CRLF_RENORMALIZE
) &&
293 has_cr_in_index(istate
, path
))
294 convert_crlf_into_lf
= 0;
296 if ((checksafe
== SAFE_CRLF_WARN
||
297 (checksafe
== SAFE_CRLF_FAIL
)) && len
) {
298 struct text_stat new_stats
;
299 memcpy(&new_stats
, &stats
, sizeof(new_stats
));
300 /* simulate "git add" */
301 if (convert_crlf_into_lf
) {
302 new_stats
.lonelf
+= new_stats
.crlf
;
305 /* simulate "git checkout" */
306 if (will_convert_lf_to_crlf(len
, &new_stats
, crlf_action
)) {
307 new_stats
.crlf
+= new_stats
.lonelf
;
308 new_stats
.lonelf
= 0;
310 check_safe_crlf(path
, crlf_action
, &stats
, &new_stats
, checksafe
);
312 if (!convert_crlf_into_lf
)
316 * At this point all of our source analysis is done, and we are sure we
317 * would convert. If we are in dry-run mode, we can give an answer.
322 /* only grow if not in place */
323 if (strbuf_avail(buf
) + buf
->len
< len
)
324 strbuf_grow(buf
, len
- buf
->len
);
326 if (crlf_action
== CRLF_AUTO
|| crlf_action
== CRLF_AUTO_INPUT
|| crlf_action
== CRLF_AUTO_CRLF
) {
328 * If we guessed, we already know we rejected a file with
329 * lone CR, and we can strip a CR without looking at what
333 unsigned char c
= *src
++;
339 unsigned char c
= *src
++;
340 if (! (c
== '\r' && (1 < len
&& *src
== '\n')))
344 strbuf_setlen(buf
, dst
- buf
->buf
);
348 static int crlf_to_worktree(const char *path
, const char *src
, size_t len
,
349 struct strbuf
*buf
, enum crlf_action crlf_action
)
351 char *to_free
= NULL
;
352 struct text_stat stats
;
354 if (!len
|| output_eol(crlf_action
) != EOL_CRLF
)
357 gather_stats(src
, len
, &stats
);
358 if (!will_convert_lf_to_crlf(len
, &stats
, crlf_action
))
361 /* are we "faking" in place editing ? */
363 to_free
= strbuf_detach(buf
, NULL
);
365 strbuf_grow(buf
, len
+ stats
.lonelf
);
367 const char *nl
= memchr(src
, '\n', len
);
370 if (nl
> src
&& nl
[-1] == '\r') {
371 strbuf_add(buf
, src
, nl
+ 1 - src
);
373 strbuf_add(buf
, src
, nl
- src
);
374 strbuf_addstr(buf
, "\r\n");
379 strbuf_add(buf
, src
, len
);
385 struct filter_params
{
393 static int filter_buffer_or_fd(int in
, int out
, void *data
)
396 * Spawn cmd and feed the buffer contents through its stdin.
398 struct child_process child_process
= CHILD_PROCESS_INIT
;
399 struct filter_params
*params
= (struct filter_params
*)data
;
400 int write_err
, status
;
401 const char *argv
[] = { NULL
, NULL
};
403 /* apply % substitution to cmd */
404 struct strbuf cmd
= STRBUF_INIT
;
405 struct strbuf path
= STRBUF_INIT
;
406 struct strbuf_expand_dict_entry dict
[] = {
411 /* quote the path to preserve spaces, etc. */
412 sq_quote_buf(&path
, params
->path
);
413 dict
[0].value
= path
.buf
;
415 /* expand all %f with the quoted path */
416 strbuf_expand(&cmd
, params
->cmd
, strbuf_expand_dict_cb
, &dict
);
417 strbuf_release(&path
);
421 child_process
.argv
= argv
;
422 child_process
.use_shell
= 1;
423 child_process
.in
= -1;
424 child_process
.out
= out
;
426 if (start_command(&child_process
))
427 return error("cannot fork to run external filter '%s'", params
->cmd
);
429 sigchain_push(SIGPIPE
, SIG_IGN
);
432 write_err
= (write_in_full(child_process
.in
,
433 params
->src
, params
->size
) < 0);
437 write_err
= copy_fd(params
->fd
, child_process
.in
);
438 if (write_err
== COPY_WRITE_ERROR
&& errno
== EPIPE
)
442 if (close(child_process
.in
))
445 error("cannot feed the input to external filter '%s'", params
->cmd
);
447 sigchain_pop(SIGPIPE
);
449 status
= finish_command(&child_process
);
451 error("external filter '%s' failed %d", params
->cmd
, status
);
453 strbuf_release(&cmd
);
454 return (write_err
|| status
);
457 static int apply_single_file_filter(const char *path
, const char *src
, size_t len
, int fd
,
458 struct strbuf
*dst
, const char *cmd
)
461 * Create a pipeline to have the command filter the buffer's
464 * (child --> cmd) --> us
467 struct strbuf nbuf
= STRBUF_INIT
;
469 struct filter_params params
;
471 memset(&async
, 0, sizeof(async
));
472 async
.proc
= filter_buffer_or_fd
;
473 async
.data
= ¶ms
;
482 if (start_async(&async
))
483 return 0; /* error was already reported */
485 if (strbuf_read(&nbuf
, async
.out
, len
) < 0) {
486 err
= error("read from external filter '%s' failed", cmd
);
488 if (close(async
.out
)) {
489 err
= error("read from external filter '%s' failed", cmd
);
491 if (finish_async(&async
)) {
492 err
= error("external filter '%s' failed", cmd
);
496 strbuf_swap(dst
, &nbuf
);
498 strbuf_release(&nbuf
);
502 #define CAP_CLEAN (1u<<0)
503 #define CAP_SMUDGE (1u<<1)
504 #define CAP_DELAY (1u<<2)
507 struct subprocess_entry subprocess
; /* must be the first member! */
508 unsigned int supported_capabilities
;
511 static int subprocess_map_initialized
;
512 static struct hashmap subprocess_map
;
514 static int start_multi_file_filter_fn(struct subprocess_entry
*subprocess
)
517 struct cmd2process
*entry
= (struct cmd2process
*)subprocess
;
518 struct string_list cap_list
= STRING_LIST_INIT_NODUP
;
520 const char *cap_name
;
521 struct child_process
*process
= &subprocess
->process
;
522 const char *cmd
= subprocess
->cmd
;
524 static const struct {
528 { "clean", CAP_CLEAN
},
529 { "smudge", CAP_SMUDGE
},
530 { "delay", CAP_DELAY
},
533 sigchain_push(SIGPIPE
, SIG_IGN
);
535 err
= packet_writel(process
->in
, "git-filter-client", "version=2", NULL
);
539 err
= strcmp(packet_read_line(process
->out
, NULL
), "git-filter-server");
541 error("external filter '%s' does not support filter protocol version 2", cmd
);
544 err
= strcmp(packet_read_line(process
->out
, NULL
), "version=2");
547 err
= packet_read_line(process
->out
, NULL
) != NULL
;
551 for (i
= 0; i
< ARRAY_SIZE(known_caps
); ++i
) {
552 err
= packet_write_fmt_gently(
553 process
->in
, "capability=%s\n", known_caps
[i
].name
);
557 err
= packet_flush_gently(process
->in
);
562 cap_buf
= packet_read_line(process
->out
, NULL
);
565 string_list_split_in_place(&cap_list
, cap_buf
, '=', 1);
567 if (cap_list
.nr
!= 2 || strcmp(cap_list
.items
[0].string
, "capability"))
570 cap_name
= cap_list
.items
[1].string
;
571 i
= ARRAY_SIZE(known_caps
) - 1;
572 while (i
>= 0 && strcmp(cap_name
, known_caps
[i
].name
))
576 entry
->supported_capabilities
|= known_caps
[i
].cap
;
578 warning("external filter '%s' requested unsupported filter capability '%s'",
581 string_list_clear(&cap_list
, 0);
585 sigchain_pop(SIGPIPE
);
590 static void handle_filter_error(const struct strbuf
*filter_status
,
591 struct cmd2process
*entry
,
592 const unsigned int wanted_capability
) {
593 if (!strcmp(filter_status
->buf
, "error"))
594 ; /* The filter signaled a problem with the file. */
595 else if (!strcmp(filter_status
->buf
, "abort") && wanted_capability
) {
597 * The filter signaled a permanent problem. Don't try to filter
598 * files with the same command for the lifetime of the current
601 entry
->supported_capabilities
&= ~wanted_capability
;
604 * Something went wrong with the protocol filter.
605 * Force shutdown and restart if another blob requires filtering.
607 error("external filter '%s' failed", entry
->subprocess
.cmd
);
608 subprocess_stop(&subprocess_map
, &entry
->subprocess
);
613 static int apply_multi_file_filter(const char *path
, const char *src
, size_t len
,
614 int fd
, struct strbuf
*dst
, const char *cmd
,
615 const unsigned int wanted_capability
,
616 struct delayed_checkout
*dco
)
620 struct cmd2process
*entry
;
621 struct child_process
*process
;
622 struct strbuf nbuf
= STRBUF_INIT
;
623 struct strbuf filter_status
= STRBUF_INIT
;
624 const char *filter_type
;
626 if (!subprocess_map_initialized
) {
627 subprocess_map_initialized
= 1;
628 hashmap_init(&subprocess_map
, (hashmap_cmp_fn
) cmd2process_cmp
,
632 entry
= (struct cmd2process
*)subprocess_find_entry(&subprocess_map
, cmd
);
638 entry
= xmalloc(sizeof(*entry
));
639 entry
->supported_capabilities
= 0;
641 if (subprocess_start(&subprocess_map
, &entry
->subprocess
, cmd
, start_multi_file_filter_fn
)) {
646 process
= &entry
->subprocess
.process
;
648 if (!(entry
->supported_capabilities
& wanted_capability
))
651 if (wanted_capability
& CAP_CLEAN
)
652 filter_type
= "clean";
653 else if (wanted_capability
& CAP_SMUDGE
)
654 filter_type
= "smudge";
656 die("unexpected filter type");
658 sigchain_push(SIGPIPE
, SIG_IGN
);
660 assert(strlen(filter_type
) < LARGE_PACKET_DATA_MAX
- strlen("command=\n"));
661 err
= packet_write_fmt_gently(process
->in
, "command=%s\n", filter_type
);
665 err
= strlen(path
) > LARGE_PACKET_DATA_MAX
- strlen("pathname=\n");
667 error("path name too long for external filter");
671 err
= packet_write_fmt_gently(process
->in
, "pathname=%s\n", path
);
675 if ((entry
->supported_capabilities
& CAP_DELAY
) &&
676 dco
&& dco
->state
== CE_CAN_DELAY
) {
678 err
= packet_write_fmt_gently(process
->in
, "can-delay=1\n");
683 err
= packet_flush_gently(process
->in
);
688 err
= write_packetized_from_fd(fd
, process
->in
);
690 err
= write_packetized_from_buf(src
, len
, process
->in
);
694 err
= subprocess_read_status(process
->out
, &filter_status
);
698 if (can_delay
&& !strcmp(filter_status
.buf
, "delayed")) {
699 string_list_insert(&dco
->filters
, cmd
);
700 string_list_insert(&dco
->paths
, path
);
702 /* The filter got the blob and wants to send us a response. */
703 err
= strcmp(filter_status
.buf
, "success");
707 err
= read_packetized_to_strbuf(process
->out
, &nbuf
) < 0;
711 err
= subprocess_read_status(process
->out
, &filter_status
);
715 err
= strcmp(filter_status
.buf
, "success");
719 sigchain_pop(SIGPIPE
);
722 handle_filter_error(&filter_status
, entry
, wanted_capability
);
724 strbuf_swap(dst
, &nbuf
);
725 strbuf_release(&nbuf
);
730 int async_query_available_blobs(const char *cmd
, struct string_list
*available_paths
)
734 struct cmd2process
*entry
;
735 struct child_process
*process
;
736 struct strbuf filter_status
= STRBUF_INIT
;
738 assert(subprocess_map_initialized
);
739 entry
= (struct cmd2process
*)subprocess_find_entry(&subprocess_map
, cmd
);
741 error("external filter '%s' is not available anymore although "
742 "not all paths have been filtered", cmd
);
745 process
= &entry
->subprocess
.process
;
746 sigchain_push(SIGPIPE
, SIG_IGN
);
748 err
= packet_write_fmt_gently(
749 process
->in
, "command=list_available_blobs\n");
753 err
= packet_flush_gently(process
->in
);
757 while ((line
= packet_read_line(process
->out
, NULL
))) {
759 if (skip_prefix(line
, "pathname=", &path
))
760 string_list_insert(available_paths
, xstrdup(path
));
762 ; /* ignore unknown keys */
765 err
= subprocess_read_status(process
->out
, &filter_status
);
769 err
= strcmp(filter_status
.buf
, "success");
772 sigchain_pop(SIGPIPE
);
775 handle_filter_error(&filter_status
, entry
, 0);
779 static struct convert_driver
{
781 struct convert_driver
*next
;
786 } *user_convert
, **user_convert_tail
;
788 static int apply_filter(const char *path
, const char *src
, size_t len
,
789 int fd
, struct strbuf
*dst
, struct convert_driver
*drv
,
790 const unsigned int wanted_capability
,
791 struct delayed_checkout
*dco
)
793 const char *cmd
= NULL
;
801 if ((wanted_capability
& CAP_CLEAN
) && !drv
->process
&& drv
->clean
)
803 else if ((wanted_capability
& CAP_SMUDGE
) && !drv
->process
&& drv
->smudge
)
807 return apply_single_file_filter(path
, src
, len
, fd
, dst
, cmd
);
808 else if (drv
->process
&& *drv
->process
)
809 return apply_multi_file_filter(path
, src
, len
, fd
, dst
,
810 drv
->process
, wanted_capability
, dco
);
815 static int read_convert_config(const char *var
, const char *value
, void *cb
)
817 const char *key
, *name
;
819 struct convert_driver
*drv
;
822 * External conversion drivers are configured using
823 * "filter.<name>.variable".
825 if (parse_config_key(var
, "filter", &name
, &namelen
, &key
) < 0 || !name
)
827 for (drv
= user_convert
; drv
; drv
= drv
->next
)
828 if (!strncmp(drv
->name
, name
, namelen
) && !drv
->name
[namelen
])
831 drv
= xcalloc(1, sizeof(struct convert_driver
));
832 drv
->name
= xmemdupz(name
, namelen
);
833 *user_convert_tail
= drv
;
834 user_convert_tail
= &(drv
->next
);
838 * filter.<name>.smudge and filter.<name>.clean specifies
843 * The command-line will not be interpolated in any way.
846 if (!strcmp("smudge", key
))
847 return git_config_string(&drv
->smudge
, var
, value
);
849 if (!strcmp("clean", key
))
850 return git_config_string(&drv
->clean
, var
, value
);
852 if (!strcmp("process", key
))
853 return git_config_string(&drv
->process
, var
, value
);
855 if (!strcmp("required", key
)) {
856 drv
->required
= git_config_bool(var
, value
);
863 static int count_ident(const char *cp
, unsigned long size
)
866 * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$"
878 if (memcmp("Id", cp
, 2))
889 * "$Id: ... "; scan up to the closing dollar sign and discard.
905 static int ident_to_git(const char *path
, const char *src
, size_t len
,
906 struct strbuf
*buf
, int ident
)
910 if (!ident
|| (src
&& !count_ident(src
, len
)))
916 /* only grow if not in place */
917 if (strbuf_avail(buf
) + buf
->len
< len
)
918 strbuf_grow(buf
, len
- buf
->len
);
921 dollar
= memchr(src
, '$', len
);
924 memmove(dst
, src
, dollar
+ 1 - src
);
925 dst
+= dollar
+ 1 - src
;
926 len
-= dollar
+ 1 - src
;
929 if (len
> 3 && !memcmp(src
, "Id:", 3)) {
930 dollar
= memchr(src
+ 3, '$', len
- 3);
933 if (memchr(src
+ 3, '\n', dollar
- src
- 3)) {
934 /* Line break before the next dollar. */
938 memcpy(dst
, "Id$", 3);
940 len
-= dollar
+ 1 - src
;
944 memmove(dst
, src
, len
);
945 strbuf_setlen(buf
, dst
+ len
- buf
->buf
);
949 static int ident_to_worktree(const char *path
, const char *src
, size_t len
,
950 struct strbuf
*buf
, int ident
)
952 unsigned char sha1
[20];
953 char *to_free
= NULL
, *dollar
, *spc
;
959 cnt
= count_ident(src
, len
);
963 /* are we "faking" in place editing ? */
965 to_free
= strbuf_detach(buf
, NULL
);
966 hash_sha1_file(src
, len
, "blob", sha1
);
968 strbuf_grow(buf
, len
+ cnt
* 43);
970 /* step 1: run to the next '$' */
971 dollar
= memchr(src
, '$', len
);
974 strbuf_add(buf
, src
, dollar
+ 1 - src
);
975 len
-= dollar
+ 1 - src
;
978 /* step 2: does it looks like a bit like Id:xxx$ or Id$ ? */
979 if (len
< 3 || memcmp("Id", src
, 2))
982 /* step 3: skip over Id$ or Id:xxxxx$ */
986 } else if (src
[2] == ':') {
988 * It's possible that an expanded Id has crept its way into the
989 * repository, we cope with that by stripping the expansion out.
990 * This is probably not a good idea, since it will cause changes
991 * on checkout, which won't go away by stash, but let's keep it
994 dollar
= memchr(src
+ 3, '$', len
- 3);
996 /* incomplete keyword, no more '$', so just quit the loop */
1000 if (memchr(src
+ 3, '\n', dollar
- src
- 3)) {
1001 /* Line break before the next dollar. */
1005 spc
= memchr(src
+ 4, ' ', dollar
- src
- 4);
1006 if (spc
&& spc
< dollar
-1) {
1007 /* There are spaces in unexpected places.
1008 * This is probably an id from some other
1009 * versioning system. Keep it for now.
1014 len
-= dollar
+ 1 - src
;
1017 /* it wasn't a "Id$" or "Id:xxxx$" */
1021 /* step 4: substitute */
1022 strbuf_addstr(buf
, "Id: ");
1023 strbuf_add(buf
, sha1_to_hex(sha1
), 40);
1024 strbuf_addstr(buf
, " $");
1026 strbuf_add(buf
, src
, len
);
1032 static enum crlf_action
git_path_check_crlf(struct attr_check_item
*check
)
1034 const char *value
= check
->value
;
1036 if (ATTR_TRUE(value
))
1038 else if (ATTR_FALSE(value
))
1040 else if (ATTR_UNSET(value
))
1042 else if (!strcmp(value
, "input"))
1043 return CRLF_TEXT_INPUT
;
1044 else if (!strcmp(value
, "auto"))
1046 return CRLF_UNDEFINED
;
1049 static enum eol
git_path_check_eol(struct attr_check_item
*check
)
1051 const char *value
= check
->value
;
1053 if (ATTR_UNSET(value
))
1055 else if (!strcmp(value
, "lf"))
1057 else if (!strcmp(value
, "crlf"))
1062 static struct convert_driver
*git_path_check_convert(struct attr_check_item
*check
)
1064 const char *value
= check
->value
;
1065 struct convert_driver
*drv
;
1067 if (ATTR_TRUE(value
) || ATTR_FALSE(value
) || ATTR_UNSET(value
))
1069 for (drv
= user_convert
; drv
; drv
= drv
->next
)
1070 if (!strcmp(value
, drv
->name
))
1075 static int git_path_check_ident(struct attr_check_item
*check
)
1077 const char *value
= check
->value
;
1079 return !!ATTR_TRUE(value
);
1083 struct convert_driver
*drv
;
1084 enum crlf_action attr_action
; /* What attr says */
1085 enum crlf_action crlf_action
; /* When no attr is set, use core.autocrlf */
1089 static void convert_attrs(struct conv_attrs
*ca
, const char *path
)
1091 static struct attr_check
*check
;
1094 check
= attr_check_initl("crlf", "ident", "filter",
1095 "eol", "text", NULL
);
1096 user_convert_tail
= &user_convert
;
1097 git_config(read_convert_config
, NULL
);
1100 if (!git_check_attr(path
, check
)) {
1101 struct attr_check_item
*ccheck
= check
->items
;
1102 ca
->crlf_action
= git_path_check_crlf(ccheck
+ 4);
1103 if (ca
->crlf_action
== CRLF_UNDEFINED
)
1104 ca
->crlf_action
= git_path_check_crlf(ccheck
+ 0);
1105 ca
->attr_action
= ca
->crlf_action
;
1106 ca
->ident
= git_path_check_ident(ccheck
+ 1);
1107 ca
->drv
= git_path_check_convert(ccheck
+ 2);
1108 if (ca
->crlf_action
!= CRLF_BINARY
) {
1109 enum eol eol_attr
= git_path_check_eol(ccheck
+ 3);
1110 if (ca
->crlf_action
== CRLF_AUTO
&& eol_attr
== EOL_LF
)
1111 ca
->crlf_action
= CRLF_AUTO_INPUT
;
1112 else if (ca
->crlf_action
== CRLF_AUTO
&& eol_attr
== EOL_CRLF
)
1113 ca
->crlf_action
= CRLF_AUTO_CRLF
;
1114 else if (eol_attr
== EOL_LF
)
1115 ca
->crlf_action
= CRLF_TEXT_INPUT
;
1116 else if (eol_attr
== EOL_CRLF
)
1117 ca
->crlf_action
= CRLF_TEXT_CRLF
;
1119 ca
->attr_action
= ca
->crlf_action
;
1122 ca
->crlf_action
= CRLF_UNDEFINED
;
1125 if (ca
->crlf_action
== CRLF_TEXT
)
1126 ca
->crlf_action
= text_eol_is_crlf() ? CRLF_TEXT_CRLF
: CRLF_TEXT_INPUT
;
1127 if (ca
->crlf_action
== CRLF_UNDEFINED
&& auto_crlf
== AUTO_CRLF_FALSE
)
1128 ca
->crlf_action
= CRLF_BINARY
;
1129 if (ca
->crlf_action
== CRLF_UNDEFINED
&& auto_crlf
== AUTO_CRLF_TRUE
)
1130 ca
->crlf_action
= CRLF_AUTO_CRLF
;
1131 if (ca
->crlf_action
== CRLF_UNDEFINED
&& auto_crlf
== AUTO_CRLF_INPUT
)
1132 ca
->crlf_action
= CRLF_AUTO_INPUT
;
1135 int would_convert_to_git_filter_fd(const char *path
)
1137 struct conv_attrs ca
;
1139 convert_attrs(&ca
, path
);
1144 * Apply a filter to an fd only if the filter is required to succeed.
1145 * We must die if the filter fails, because the original data before
1146 * filtering is not available.
1148 if (!ca
.drv
->required
)
1151 return apply_filter(path
, NULL
, 0, -1, NULL
, ca
.drv
, CAP_CLEAN
, NULL
);
1154 const char *get_convert_attr_ascii(const char *path
)
1156 struct conv_attrs ca
;
1158 convert_attrs(&ca
, path
);
1159 switch (ca
.attr_action
) {
1160 case CRLF_UNDEFINED
:
1166 case CRLF_TEXT_INPUT
:
1167 return "text eol=lf";
1168 case CRLF_TEXT_CRLF
:
1169 return "text eol=crlf";
1172 case CRLF_AUTO_CRLF
:
1173 return "text=auto eol=crlf";
1174 case CRLF_AUTO_INPUT
:
1175 return "text=auto eol=lf";
1180 int convert_to_git(const struct index_state
*istate
,
1181 const char *path
, const char *src
, size_t len
,
1182 struct strbuf
*dst
, enum safe_crlf checksafe
)
1185 struct conv_attrs ca
;
1187 convert_attrs(&ca
, path
);
1189 ret
|= apply_filter(path
, src
, len
, -1, dst
, ca
.drv
, CAP_CLEAN
, NULL
);
1190 if (!ret
&& ca
.drv
&& ca
.drv
->required
)
1191 die("%s: clean filter '%s' failed", path
, ca
.drv
->name
);
1197 ret
|= crlf_to_git(istate
, path
, src
, len
, dst
, ca
.crlf_action
, checksafe
);
1202 return ret
| ident_to_git(path
, src
, len
, dst
, ca
.ident
);
1205 void convert_to_git_filter_fd(const struct index_state
*istate
,
1206 const char *path
, int fd
, struct strbuf
*dst
,
1207 enum safe_crlf checksafe
)
1209 struct conv_attrs ca
;
1210 convert_attrs(&ca
, path
);
1213 assert(ca
.drv
->clean
|| ca
.drv
->process
);
1215 if (!apply_filter(path
, NULL
, 0, fd
, dst
, ca
.drv
, CAP_CLEAN
, NULL
))
1216 die("%s: clean filter '%s' failed", path
, ca
.drv
->name
);
1218 crlf_to_git(istate
, path
, dst
->buf
, dst
->len
, dst
, ca
.crlf_action
, checksafe
);
1219 ident_to_git(path
, dst
->buf
, dst
->len
, dst
, ca
.ident
);
1222 static int convert_to_working_tree_internal(const char *path
, const char *src
,
1223 size_t len
, struct strbuf
*dst
,
1224 int normalizing
, struct delayed_checkout
*dco
)
1226 int ret
= 0, ret_filter
= 0;
1227 struct conv_attrs ca
;
1229 convert_attrs(&ca
, path
);
1231 ret
|= ident_to_worktree(path
, src
, len
, dst
, ca
.ident
);
1237 * CRLF conversion can be skipped if normalizing, unless there
1238 * is a smudge or process filter (even if the process filter doesn't
1239 * support smudge). The filters might expect CRLFs.
1241 if ((ca
.drv
&& (ca
.drv
->smudge
|| ca
.drv
->process
)) || !normalizing
) {
1242 ret
|= crlf_to_worktree(path
, src
, len
, dst
, ca
.crlf_action
);
1249 ret_filter
= apply_filter(
1250 path
, src
, len
, -1, dst
, ca
.drv
, CAP_SMUDGE
, dco
);
1251 if (!ret_filter
&& ca
.drv
&& ca
.drv
->required
)
1252 die("%s: smudge filter %s failed", path
, ca
.drv
->name
);
1254 return ret
| ret_filter
;
1257 int async_convert_to_working_tree(const char *path
, const char *src
,
1258 size_t len
, struct strbuf
*dst
,
1261 return convert_to_working_tree_internal(path
, src
, len
, dst
, 0, dco
);
1264 int convert_to_working_tree(const char *path
, const char *src
, size_t len
, struct strbuf
*dst
)
1266 return convert_to_working_tree_internal(path
, src
, len
, dst
, 0, NULL
);
1269 int renormalize_buffer(const struct index_state
*istate
, const char *path
,
1270 const char *src
, size_t len
, struct strbuf
*dst
)
1272 int ret
= convert_to_working_tree_internal(path
, src
, len
, dst
, 1, NULL
);
1277 return ret
| convert_to_git(istate
, path
, src
, len
, dst
, SAFE_CRLF_RENORMALIZE
);
1280 /*****************************************************************
1282 * Streaming conversion support
1284 *****************************************************************/
1286 typedef int (*filter_fn
)(struct stream_filter
*,
1287 const char *input
, size_t *isize_p
,
1288 char *output
, size_t *osize_p
);
1289 typedef void (*free_fn
)(struct stream_filter
*);
1291 struct stream_filter_vtbl
{
1296 struct stream_filter
{
1297 struct stream_filter_vtbl
*vtbl
;
1300 static int null_filter_fn(struct stream_filter
*filter
,
1301 const char *input
, size_t *isize_p
,
1302 char *output
, size_t *osize_p
)
1307 return 0; /* we do not keep any states */
1309 if (*osize_p
< count
)
1312 memmove(output
, input
, count
);
1319 static void null_free_fn(struct stream_filter
*filter
)
1321 ; /* nothing -- null instances are shared */
1324 static struct stream_filter_vtbl null_vtbl
= {
1329 static struct stream_filter null_filter_singleton
= {
1333 int is_null_stream_filter(struct stream_filter
*filter
)
1335 return filter
== &null_filter_singleton
;
1343 struct lf_to_crlf_filter
{
1344 struct stream_filter filter
;
1345 unsigned has_held
:1;
1349 static int lf_to_crlf_filter_fn(struct stream_filter
*filter
,
1350 const char *input
, size_t *isize_p
,
1351 char *output
, size_t *osize_p
)
1353 size_t count
, o
= 0;
1354 struct lf_to_crlf_filter
*lf_to_crlf
= (struct lf_to_crlf_filter
*)filter
;
1357 * We may be holding onto the CR to see if it is followed by a
1358 * LF, in which case we would need to go to the main loop.
1359 * Otherwise, just emit it to the output stream.
1361 if (lf_to_crlf
->has_held
&& (lf_to_crlf
->held
!= '\r' || !input
)) {
1362 output
[o
++] = lf_to_crlf
->held
;
1363 lf_to_crlf
->has_held
= 0;
1366 /* We are told to drain */
1373 if (count
|| lf_to_crlf
->has_held
) {
1377 if (lf_to_crlf
->has_held
) {
1379 lf_to_crlf
->has_held
= 0;
1382 for (i
= 0; o
< *osize_p
&& i
< count
; i
++) {
1387 } else if (was_cr
) {
1389 * Previous round saw CR and it is not followed
1390 * by a LF; emit the CR before processing the
1391 * current character.
1397 * We may have consumed the last output slot,
1398 * in which case we need to break out of this
1399 * loop; hold the current character before
1402 if (*osize_p
<= o
) {
1403 lf_to_crlf
->has_held
= 1;
1404 lf_to_crlf
->held
= ch
;
1405 continue; /* break but increment i */
1420 if (!lf_to_crlf
->has_held
&& was_cr
) {
1421 lf_to_crlf
->has_held
= 1;
1422 lf_to_crlf
->held
= '\r';
1428 static void lf_to_crlf_free_fn(struct stream_filter
*filter
)
1433 static struct stream_filter_vtbl lf_to_crlf_vtbl
= {
1434 lf_to_crlf_filter_fn
,
1438 static struct stream_filter
*lf_to_crlf_filter(void)
1440 struct lf_to_crlf_filter
*lf_to_crlf
= xcalloc(1, sizeof(*lf_to_crlf
));
1442 lf_to_crlf
->filter
.vtbl
= &lf_to_crlf_vtbl
;
1443 return (struct stream_filter
*)lf_to_crlf
;
1449 #define FILTER_BUFFER 1024
1450 struct cascade_filter
{
1451 struct stream_filter filter
;
1452 struct stream_filter
*one
;
1453 struct stream_filter
*two
;
1454 char buf
[FILTER_BUFFER
];
1458 static int cascade_filter_fn(struct stream_filter
*filter
,
1459 const char *input
, size_t *isize_p
,
1460 char *output
, size_t *osize_p
)
1462 struct cascade_filter
*cas
= (struct cascade_filter
*) filter
;
1464 size_t sz
= *osize_p
;
1465 size_t to_feed
, remaining
;
1468 * input -- (one) --> buf -- (two) --> output
1470 while (filled
< sz
) {
1471 remaining
= sz
- filled
;
1473 /* do we already have something to feed two with? */
1474 if (cas
->ptr
< cas
->end
) {
1475 to_feed
= cas
->end
- cas
->ptr
;
1476 if (stream_filter(cas
->two
,
1477 cas
->buf
+ cas
->ptr
, &to_feed
,
1478 output
+ filled
, &remaining
))
1480 cas
->ptr
+= (cas
->end
- cas
->ptr
) - to_feed
;
1481 filled
= sz
- remaining
;
1485 /* feed one from upstream and have it emit into our buffer */
1486 to_feed
= input
? *isize_p
: 0;
1487 if (input
&& !to_feed
)
1489 remaining
= sizeof(cas
->buf
);
1490 if (stream_filter(cas
->one
,
1492 cas
->buf
, &remaining
))
1494 cas
->end
= sizeof(cas
->buf
) - remaining
;
1497 size_t fed
= *isize_p
- to_feed
;
1502 /* do we know that we drained one completely? */
1503 if (input
|| cas
->end
)
1506 /* tell two to drain; we have nothing more to give it */
1508 remaining
= sz
- filled
;
1509 if (stream_filter(cas
->two
,
1511 output
+ filled
, &remaining
))
1513 if (remaining
== (sz
- filled
))
1514 break; /* completely drained two */
1515 filled
= sz
- remaining
;
1521 static void cascade_free_fn(struct stream_filter
*filter
)
1523 struct cascade_filter
*cas
= (struct cascade_filter
*)filter
;
1524 free_stream_filter(cas
->one
);
1525 free_stream_filter(cas
->two
);
1529 static struct stream_filter_vtbl cascade_vtbl
= {
1534 static struct stream_filter
*cascade_filter(struct stream_filter
*one
,
1535 struct stream_filter
*two
)
1537 struct cascade_filter
*cascade
;
1539 if (!one
|| is_null_stream_filter(one
))
1541 if (!two
|| is_null_stream_filter(two
))
1544 cascade
= xmalloc(sizeof(*cascade
));
1547 cascade
->end
= cascade
->ptr
= 0;
1548 cascade
->filter
.vtbl
= &cascade_vtbl
;
1549 return (struct stream_filter
*)cascade
;
1555 #define IDENT_DRAINING (-1)
1556 #define IDENT_SKIPPING (-2)
1557 struct ident_filter
{
1558 struct stream_filter filter
;
1561 char ident
[45]; /* ": x40 $" */
1564 static int is_foreign_ident(const char *str
)
1568 if (!skip_prefix(str
, "$Id: ", &str
))
1570 for (i
= 0; str
[i
]; i
++) {
1571 if (isspace(str
[i
]) && str
[i
+1] != '$')
1577 static void ident_drain(struct ident_filter
*ident
, char **output_p
, size_t *osize_p
)
1579 size_t to_drain
= ident
->left
.len
;
1581 if (*osize_p
< to_drain
)
1582 to_drain
= *osize_p
;
1584 memcpy(*output_p
, ident
->left
.buf
, to_drain
);
1585 strbuf_remove(&ident
->left
, 0, to_drain
);
1586 *output_p
+= to_drain
;
1587 *osize_p
-= to_drain
;
1589 if (!ident
->left
.len
)
1593 static int ident_filter_fn(struct stream_filter
*filter
,
1594 const char *input
, size_t *isize_p
,
1595 char *output
, size_t *osize_p
)
1597 struct ident_filter
*ident
= (struct ident_filter
*)filter
;
1598 static const char head
[] = "$Id";
1601 /* drain upon eof */
1602 switch (ident
->state
) {
1604 strbuf_add(&ident
->left
, head
, ident
->state
);
1605 case IDENT_SKIPPING
:
1607 case IDENT_DRAINING
:
1608 ident_drain(ident
, &output
, osize_p
);
1613 while (*isize_p
|| (ident
->state
== IDENT_DRAINING
)) {
1616 if (ident
->state
== IDENT_DRAINING
) {
1617 ident_drain(ident
, &output
, osize_p
);
1626 if (ident
->state
== IDENT_SKIPPING
) {
1628 * Skipping until '$' or LF, but keeping them
1629 * in case it is a foreign ident.
1631 strbuf_addch(&ident
->left
, ch
);
1632 if (ch
!= '\n' && ch
!= '$')
1634 if (ch
== '$' && !is_foreign_ident(ident
->left
.buf
)) {
1635 strbuf_setlen(&ident
->left
, sizeof(head
) - 1);
1636 strbuf_addstr(&ident
->left
, ident
->ident
);
1638 ident
->state
= IDENT_DRAINING
;
1642 if (ident
->state
< sizeof(head
) &&
1643 head
[ident
->state
] == ch
) {
1649 strbuf_add(&ident
->left
, head
, ident
->state
);
1650 if (ident
->state
== sizeof(head
) - 1) {
1651 if (ch
!= ':' && ch
!= '$') {
1652 strbuf_addch(&ident
->left
, ch
);
1658 strbuf_addch(&ident
->left
, ch
);
1659 ident
->state
= IDENT_SKIPPING
;
1661 strbuf_addstr(&ident
->left
, ident
->ident
);
1662 ident
->state
= IDENT_DRAINING
;
1667 strbuf_addch(&ident
->left
, ch
);
1668 ident
->state
= IDENT_DRAINING
;
1673 static void ident_free_fn(struct stream_filter
*filter
)
1675 struct ident_filter
*ident
= (struct ident_filter
*)filter
;
1676 strbuf_release(&ident
->left
);
1680 static struct stream_filter_vtbl ident_vtbl
= {
1685 static struct stream_filter
*ident_filter(const unsigned char *sha1
)
1687 struct ident_filter
*ident
= xmalloc(sizeof(*ident
));
1689 xsnprintf(ident
->ident
, sizeof(ident
->ident
),
1690 ": %s $", sha1_to_hex(sha1
));
1691 strbuf_init(&ident
->left
, 0);
1692 ident
->filter
.vtbl
= &ident_vtbl
;
1694 return (struct stream_filter
*)ident
;
1698 * Return an appropriately constructed filter for the path, or NULL if
1699 * the contents cannot be filtered without reading the whole thing
1702 * Note that you would be crazy to set CRLF, smuge/clean or ident to a
1703 * large binary blob you would want us not to slurp into the memory!
1705 struct stream_filter
*get_stream_filter(const char *path
, const unsigned char *sha1
)
1707 struct conv_attrs ca
;
1708 struct stream_filter
*filter
= NULL
;
1710 convert_attrs(&ca
, path
);
1711 if (ca
.drv
&& (ca
.drv
->process
|| ca
.drv
->smudge
|| ca
.drv
->clean
))
1714 if (ca
.crlf_action
== CRLF_AUTO
|| ca
.crlf_action
== CRLF_AUTO_CRLF
)
1718 filter
= ident_filter(sha1
);
1720 if (output_eol(ca
.crlf_action
) == EOL_CRLF
)
1721 filter
= cascade_filter(filter
, lf_to_crlf_filter());
1723 filter
= cascade_filter(filter
, &null_filter_singleton
);
1728 void free_stream_filter(struct stream_filter
*filter
)
1730 filter
->vtbl
->free(filter
);
1733 int stream_filter(struct stream_filter
*filter
,
1734 const char *input
, size_t *isize_p
,
1735 char *output
, size_t *osize_p
)
1737 return filter
->vtbl
->filter(filter
, input
, isize_p
, output
, osize_p
);