]> git.ipfire.org Git - thirdparty/git.git/blob - convert.c
projects / thirdparty / git.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
treewide: be explicit about dependence on gettext.h
[thirdparty/git.git] / convert.c
1 #include "cache.h"
2 #include "config.h"
3 #include "gettext.h"
4 #include "hex.h"
5 #include "object-store.h"
6 #include "attr.h"
7 #include "run-command.h"
8 #include "quote.h"
9 #include "sigchain.h"
10 #include "pkt-line.h"
11 #include "sub-process.h"
12 #include "utf8.h"
13 #include "ll-merge.h"
14
15 /*
16 * convert.c - convert a file when checking it out and checking it in.
17 *
18 * This should use the pathname to decide on whether it wants to do some
19 * more interesting conversions (automatic gzip/unzip, general format
20 * conversions etc etc), but by default it just does automatic CRLF<->LF
21 * translation when the "text" attribute or "auto_crlf" option is set.
22 */
23
24 /* Stat bits: When BIN is set, the txt bits are unset */
25 #define CONVERT_STAT_BITS_TXT_LF 0x1
26 #define CONVERT_STAT_BITS_TXT_CRLF 0x2
27 #define CONVERT_STAT_BITS_BIN 0x4
28
29 struct text_stat {
30 /* NUL, CR, LF and CRLF counts */
31 unsigned nul, lonecr, lonelf, crlf;
32
33 /* These are just approximations! */
34 unsigned printable, nonprintable;
35 };
36
37 static void gather_stats(const char *buf, unsigned long size, struct text_stat *stats)
38 {
39 unsigned long i;
40
41 memset(stats, 0, sizeof(*stats));
42
43 for (i = 0; i < size; i++) {
44 unsigned char c = buf[i];
45 if (c == '\r') {
46 if (i+1 < size && buf[i+1] == '\n') {
47 stats->crlf++;
48 i++;
49 } else
50 stats->lonecr++;
51 continue;
52 }
53 if (c == '\n') {
54 stats->lonelf++;
55 continue;
56 }
57 if (c == 127)
58 /* DEL */
59 stats->nonprintable++;
60 else if (c < 32) {
61 switch (c) {
62 /* BS, HT, ESC and FF */
63 case '\b': case '\t': case '\033': case '\014':
64 stats->printable++;
65 break;
66 case 0:
67 stats->nul++;
68 /* fall through */
69 default:
70 stats->nonprintable++;
71 }
72 }
73 else
74 stats->printable++;
75 }
76
77 /* If file ends with EOF then don't count this EOF as non-printable. */
78 if (size >= 1 && buf[size-1] == '\032')
79 stats->nonprintable--;
80 }
81
82 /*
83 * The same heuristics as diff.c::mmfile_is_binary()
84 * We treat files with bare CR as binary
85 */
86 static int convert_is_binary(const struct text_stat *stats)
87 {
88 if (stats->lonecr)
89 return 1;
90 if (stats->nul)
91 return 1;
92 if ((stats->printable >> 7) < stats->nonprintable)
93 return 1;
94 return 0;
95 }
96
97 static unsigned int gather_convert_stats(const char *data, unsigned long size)
98 {
99 struct text_stat stats;
100 int ret = 0;
101 if (!data || !size)
102 return 0;
103 gather_stats(data, size, &stats);
104 if (convert_is_binary(&stats))
105 ret |= CONVERT_STAT_BITS_BIN;
106 if (stats.crlf)
107 ret |= CONVERT_STAT_BITS_TXT_CRLF;
108 if (stats.lonelf)
109 ret |= CONVERT_STAT_BITS_TXT_LF;
110
111 return ret;
112 }
113
114 static const char *gather_convert_stats_ascii(const char *data, unsigned long size)
115 {
116 unsigned int convert_stats = gather_convert_stats(data, size);
117
118 if (convert_stats & CONVERT_STAT_BITS_BIN)
119 return "-text";
120 switch (convert_stats) {
121 case CONVERT_STAT_BITS_TXT_LF:
122 return "lf";
123 case CONVERT_STAT_BITS_TXT_CRLF:
124 return "crlf";
125 case CONVERT_STAT_BITS_TXT_LF | CONVERT_STAT_BITS_TXT_CRLF:
126 return "mixed";
127 default:
128 return "none";
129 }
130 }
131
132 const char *get_cached_convert_stats_ascii(struct index_state *istate,
133 const char *path)
134 {
135 const char *ret;
136 unsigned long sz;
137 void *data = read_blob_data_from_index(istate, path, &sz);
138 ret = gather_convert_stats_ascii(data, sz);
139 free(data);
140 return ret;
141 }
142
143 const char *get_wt_convert_stats_ascii(const char *path)
144 {
145 const char *ret = "";
146 struct strbuf sb = STRBUF_INIT;
147 if (strbuf_read_file(&sb, path, 0) >= 0)
148 ret = gather_convert_stats_ascii(sb.buf, sb.len);
149 strbuf_release(&sb);
150 return ret;
151 }
152
153 static int text_eol_is_crlf(void)
154 {
155 if (auto_crlf == AUTO_CRLF_TRUE)
156 return 1;
157 else if (auto_crlf == AUTO_CRLF_INPUT)
158 return 0;
159 if (core_eol == EOL_CRLF)
160 return 1;
161 if (core_eol == EOL_UNSET && EOL_NATIVE == EOL_CRLF)
162 return 1;
163 return 0;
164 }
165
166 static enum eol output_eol(enum convert_crlf_action crlf_action)
167 {
168 switch (crlf_action) {
169 case CRLF_BINARY:
170 return EOL_UNSET;
171 case CRLF_TEXT_CRLF:
172 return EOL_CRLF;
173 case CRLF_TEXT_INPUT:
174 return EOL_LF;
175 case CRLF_UNDEFINED:
176 case CRLF_AUTO_CRLF:
177 return EOL_CRLF;
178 case CRLF_AUTO_INPUT:
179 return EOL_LF;
180 case CRLF_TEXT:
181 case CRLF_AUTO:
182 /* fall through */
183 return text_eol_is_crlf() ? EOL_CRLF : EOL_LF;
184 }
185 warning(_("illegal crlf_action %d"), (int)crlf_action);
186 return core_eol;
187 }
188
189 static void check_global_conv_flags_eol(const char *path,
190 struct text_stat *old_stats, struct text_stat *new_stats,
191 int conv_flags)
192 {
193 if (old_stats->crlf && !new_stats->crlf ) {
194 /*
195 * CRLFs would not be restored by checkout
196 */
197 if (conv_flags & CONV_EOL_RNDTRP_DIE)
198 die(_("CRLF would be replaced by LF in %s"), path);
199 else if (conv_flags & CONV_EOL_RNDTRP_WARN)
200 warning(_("in the working copy of '%s', CRLF will be"
201 " replaced by LF the next time Git touches"
202 " it"), path);
203 } else if (old_stats->lonelf && !new_stats->lonelf ) {
204 /*
205 * CRLFs would be added by checkout
206 */
207 if (conv_flags & CONV_EOL_RNDTRP_DIE)
208 die(_("LF would be replaced by CRLF in %s"), path);
209 else if (conv_flags & CONV_EOL_RNDTRP_WARN)
210 warning(_("in the working copy of '%s', LF will be"
211 " replaced by CRLF the next time Git touches"
212 " it"), path);
213 }
214 }
215
216 static int has_crlf_in_index(struct index_state *istate, const char *path)
217 {
218 unsigned long sz;
219 void *data;
220 const char *crp;
221 int has_crlf = 0;
222
223 data = read_blob_data_from_index(istate, path, &sz);
224 if (!data)
225 return 0;
226
227 crp = memchr(data, '\r', sz);
228 if (crp) {
229 unsigned int ret_stats;
230 ret_stats = gather_convert_stats(data, sz);
231 if (!(ret_stats & CONVERT_STAT_BITS_BIN) &&
232 (ret_stats & CONVERT_STAT_BITS_TXT_CRLF))
233 has_crlf = 1;
234 }
235 free(data);
236 return has_crlf;
237 }
238
239 static int will_convert_lf_to_crlf(struct text_stat *stats,
240 enum convert_crlf_action crlf_action)
241 {
242 if (output_eol(crlf_action) != EOL_CRLF)
243 return 0;
244 /* No "naked" LF? Nothing to convert, regardless. */
245 if (!stats->lonelf)
246 return 0;
247
248 if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
249 /* If we have any CR or CRLF line endings, we do not touch it */
250 /* This is the new safer autocrlf-handling */
251 if (stats->lonecr || stats->crlf)
252 return 0;
253
254 if (convert_is_binary(stats))
255 return 0;
256 }
257 return 1;
258
259 }
260
261 static int validate_encoding(const char *path, const char *enc,
262 const char *data, size_t len, int die_on_error)
263 {
264 const char *stripped;
265
266 /* We only check for UTF here as UTF?? can be an alias for UTF-?? */
267 if (skip_iprefix(enc, "UTF", &stripped)) {
268 skip_prefix(stripped, "-", &stripped);
269
270 /*
271 * Check for detectable errors in UTF encodings
272 */
273 if (has_prohibited_utf_bom(enc, data, len)) {
274 const char *error_msg = _(
275 "BOM is prohibited in '%s' if encoded as %s");
276 /*
277 * This advice is shown for UTF-??BE and UTF-??LE encodings.
278 * We cut off the last two characters of the encoding name
279 * to generate the encoding name suitable for BOMs.
280 */
281 const char *advise_msg = _(
282 "The file '%s' contains a byte order "
283 "mark (BOM). Please use UTF-%.*s as "
284 "working-tree-encoding.");
285 int stripped_len = strlen(stripped) - strlen("BE");
286 advise(advise_msg, path, stripped_len, stripped);
287 if (die_on_error)
288 die(error_msg, path, enc);
289 else {
290 return error(error_msg, path, enc);
291 }
292
293 } else if (is_missing_required_utf_bom(enc, data, len)) {
294 const char *error_msg = _(
295 "BOM is required in '%s' if encoded as %s");
296 const char *advise_msg = _(
297 "The file '%s' is missing a byte order "
298 "mark (BOM). Please use UTF-%sBE or UTF-%sLE "
299 "(depending on the byte order) as "
300 "working-tree-encoding.");
301 advise(advise_msg, path, stripped, stripped);
302 if (die_on_error)
303 die(error_msg, path, enc);
304 else {
305 return error(error_msg, path, enc);
306 }
307 }
308
309 }
310 return 0;
311 }
312
313 static void trace_encoding(const char *context, const char *path,
314 const char *encoding, const char *buf, size_t len)
315 {
316 static struct trace_key coe = TRACE_KEY_INIT(WORKING_TREE_ENCODING);
317 struct strbuf trace = STRBUF_INIT;
318 int i;
319
320 strbuf_addf(&trace, "%s (%s, considered %s):\n", context, path, encoding);
321 for (i = 0; i < len && buf; ++i) {
322 strbuf_addf(
323 &trace, "| \033[2m%2i:\033[0m %2x \033[2m%c\033[0m%c",
324 i,
325 (unsigned char) buf[i],
326 (buf[i] > 32 && buf[i] < 127 ? buf[i] : ' '),
327 ((i+1) % 8 && (i+1) < len ? ' ' : '\n')
328 );
329 }
330 strbuf_addchars(&trace, '\n', 1);
331
332 trace_strbuf(&coe, &trace);
333 strbuf_release(&trace);
334 }
335
336 static int check_roundtrip(const char *enc_name)
337 {
338 /*
339 * check_roundtrip_encoding contains a string of comma and/or
340 * space separated encodings (eg. "UTF-16, ASCII, CP1125").
341 * Search for the given encoding in that string.
342 */
343 const char *found = strcasestr(check_roundtrip_encoding, enc_name);
344 const char *next;
345 int len;
346 if (!found)
347 return 0;
348 next = found + strlen(enc_name);
349 len = strlen(check_roundtrip_encoding);
350 return (found && (
351 /*
352 * check that the found encoding is at the
353 * beginning of check_roundtrip_encoding or
354 * that it is prefixed with a space or comma
355 */
356 found == check_roundtrip_encoding || (
357 (isspace(found[-1]) || found[-1] == ',')
358 )
359 ) && (
360 /*
361 * check that the found encoding is at the
362 * end of check_roundtrip_encoding or
363 * that it is suffixed with a space or comma
364 */
365 next == check_roundtrip_encoding + len || (
366 next < check_roundtrip_encoding + len &&
367 (isspace(next[0]) || next[0] == ',')
368 )
369 ));
370 }
371
372 static const char *default_encoding = "UTF-8";
373
374 static int encode_to_git(const char *path, const char *src, size_t src_len,
375 struct strbuf *buf, const char *enc, int conv_flags)
376 {
377 char *dst;
378 size_t dst_len;
379 int die_on_error = conv_flags & CONV_WRITE_OBJECT;
380
381 /*
382 * No encoding is specified or there is nothing to encode.
383 * Tell the caller that the content was not modified.
384 */
385 if (!enc || (src && !src_len))
386 return 0;
387
388 /*
389 * Looks like we got called from "would_convert_to_git()".
390 * This means Git wants to know if it would encode (= modify!)
391 * the content. Let's answer with "yes", since an encoding was
392 * specified.
393 */
394 if (!buf && !src)
395 return 1;
396
397 if (validate_encoding(path, enc, src, src_len, die_on_error))
398 return 0;
399
400 trace_encoding("source", path, enc, src, src_len);
401 dst = reencode_string_len(src, src_len, default_encoding, enc,
402 &dst_len);
403 if (!dst) {
404 /*
405 * We could add the blob "as-is" to Git. However, on checkout
406 * we would try to re-encode to the original encoding. This
407 * would fail and we would leave the user with a messed-up
408 * working tree. Let's try to avoid this by screaming loud.
409 */
410 const char* msg = _("failed to encode '%s' from %s to %s");
411 if (die_on_error)
412 die(msg, path, enc, default_encoding);
413 else {
414 error(msg, path, enc, default_encoding);
415 return 0;
416 }
417 }
418 trace_encoding("destination", path, default_encoding, dst, dst_len);
419
420 /*
421 * UTF supports lossless conversion round tripping [1] and conversions
422 * between UTF and other encodings are mostly round trip safe as
423 * Unicode aims to be a superset of all other character encodings.
424 * However, certain encodings (e.g. SHIFT-JIS) are known to have round
425 * trip issues [2]. Check the round trip conversion for all encodings
426 * listed in core.checkRoundtripEncoding.
427 *
428 * The round trip check is only performed if content is written to Git.
429 * This ensures that no information is lost during conversion to/from
430 * the internal UTF-8 representation.
431 *
432 * Please note, the code below is not tested because I was not able to
433 * generate a faulty round trip without an iconv error. Iconv errors
434 * are already caught above.
435 *
436 * [1] http://unicode.org/faq/utf_bom.html#gen2
437 * [2] https://support.microsoft.com/en-us/help/170559/prb-conversion-problem-between-shift-jis-and-unicode
438 */
439 if (die_on_error && check_roundtrip(enc)) {
440 char *re_src;
441 size_t re_src_len;
442
443 re_src = reencode_string_len(dst, dst_len,
444 enc, default_encoding,
445 &re_src_len);
446
447 trace_printf("Checking roundtrip encoding for %s...\n", enc);
448 trace_encoding("reencoded source", path, enc,
449 re_src, re_src_len);
450
451 if (!re_src || src_len != re_src_len ||
452 memcmp(src, re_src, src_len)) {
453 const char* msg = _("encoding '%s' from %s to %s and "
454 "back is not the same");
455 die(msg, path, enc, default_encoding);
456 }
457
458 free(re_src);
459 }
460
461 strbuf_attach(buf, dst, dst_len, dst_len + 1);
462 return 1;
463 }
464
465 static int encode_to_worktree(const char *path, const char *src, size_t src_len,
466 struct strbuf *buf, const char *enc)
467 {
468 char *dst;
469 size_t dst_len;
470
471 /*
472 * No encoding is specified or there is nothing to encode.
473 * Tell the caller that the content was not modified.
474 */
475 if (!enc || (src && !src_len))
476 return 0;
477
478 dst = reencode_string_len(src, src_len, enc, default_encoding,
479 &dst_len);
480 if (!dst) {
481 error(_("failed to encode '%s' from %s to %s"),
482 path, default_encoding, enc);
483 return 0;
484 }
485
486 strbuf_attach(buf, dst, dst_len, dst_len + 1);
487 return 1;
488 }
489
490 static int crlf_to_git(struct index_state *istate,
491 const char *path, const char *src, size_t len,
492 struct strbuf *buf,
493 enum convert_crlf_action crlf_action, int conv_flags)
494 {
495 struct text_stat stats;
496 char *dst;
497 int convert_crlf_into_lf;
498
499 if (crlf_action == CRLF_BINARY ||
500 (src && !len))
501 return 0;
502
503 /*
504 * If we are doing a dry-run and have no source buffer, there is
505 * nothing to analyze; we must assume we would convert.
506 */
507 if (!buf && !src)
508 return 1;
509
510 gather_stats(src, len, &stats);
511 /* Optimization: No CRLF? Nothing to convert, regardless. */
512 convert_crlf_into_lf = !!stats.crlf;
513
514 if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
515 if (convert_is_binary(&stats))
516 return 0;
517 /*
518 * If the file in the index has any CR in it, do not
519 * convert. This is the new safer autocrlf handling,
520 * unless we want to renormalize in a merge or
521 * cherry-pick.
522 */
523 if ((!(conv_flags & CONV_EOL_RENORMALIZE)) &&
524 has_crlf_in_index(istate, path))
525 convert_crlf_into_lf = 0;
526 }
527 if (((conv_flags & CONV_EOL_RNDTRP_WARN) ||
528 ((conv_flags & CONV_EOL_RNDTRP_DIE) && len))) {
529 struct text_stat new_stats;
530 memcpy(&new_stats, &stats, sizeof(new_stats));
531 /* simulate "git add" */
532 if (convert_crlf_into_lf) {
533 new_stats.lonelf += new_stats.crlf;
534 new_stats.crlf = 0;
535 }
536 /* simulate "git checkout" */
537 if (will_convert_lf_to_crlf(&new_stats, crlf_action)) {
538 new_stats.crlf += new_stats.lonelf;
539 new_stats.lonelf = 0;
540 }
541 check_global_conv_flags_eol(path, &stats, &new_stats, conv_flags);
542 }
543 if (!convert_crlf_into_lf)
544 return 0;
545
546 /*
547 * At this point all of our source analysis is done, and we are sure we
548 * would convert. If we are in dry-run mode, we can give an answer.
549 */
550 if (!buf)
551 return 1;
552
553 /* only grow if not in place */
554 if (strbuf_avail(buf) + buf->len < len)
555 strbuf_grow(buf, len - buf->len);
556 dst = buf->buf;
557 if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
558 /*
559 * If we guessed, we already know we rejected a file with
560 * lone CR, and we can strip a CR without looking at what
561 * follow it.
562 */
563 do {
564 unsigned char c = *src++;
565 if (c != '\r')
566 *dst++ = c;
567 } while (--len);
568 } else {
569 do {
570 unsigned char c = *src++;
571 if (! (c == '\r' && (1 < len && *src == '\n')))
572 *dst++ = c;
573 } while (--len);
574 }
575 strbuf_setlen(buf, dst - buf->buf);
576 return 1;
577 }
578
579 static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf,
580 enum convert_crlf_action crlf_action)
581 {
582 char *to_free = NULL;
583 struct text_stat stats;
584
585 if (!len || output_eol(crlf_action) != EOL_CRLF)
586 return 0;
587
588 gather_stats(src, len, &stats);
589 if (!will_convert_lf_to_crlf(&stats, crlf_action))
590 return 0;
591
592 /* are we "faking" in place editing ? */
593 if (src == buf->buf)
594 to_free = strbuf_detach(buf, NULL);
595
596 strbuf_grow(buf, len + stats.lonelf);
597 for (;;) {
598 const char *nl = memchr(src, '\n', len);
599 if (!nl)
600 break;
601 if (nl > src && nl[-1] == '\r') {
602 strbuf_add(buf, src, nl + 1 - src);
603 } else {
604 strbuf_add(buf, src, nl - src);
605 strbuf_addstr(buf, "\r\n");
606 }
607 len -= nl + 1 - src;
608 src = nl + 1;
609 }
610 strbuf_add(buf, src, len);
611
612 free(to_free);
613 return 1;
614 }
615
616 struct filter_params {
617 const char *src;
618 size_t size;
619 int fd;
620 const char *cmd;
621 const char *path;
622 };
623
624 static int filter_buffer_or_fd(int in UNUSED, int out, void *data)
625 {
626 /*
627 * Spawn cmd and feed the buffer contents through its stdin.
628 */
629 struct child_process child_process = CHILD_PROCESS_INIT;
630 struct filter_params *params = (struct filter_params *)data;
631 int write_err, status;
632
633 /* apply % substitution to cmd */
634 struct strbuf cmd = STRBUF_INIT;
635 struct strbuf path = STRBUF_INIT;
636 struct strbuf_expand_dict_entry dict[] = {
637 { "f", NULL, },
638 { NULL, NULL, },
639 };
640
641 /* quote the path to preserve spaces, etc. */
642 sq_quote_buf(&path, params->path);
643 dict[0].value = path.buf;
644
645 /* expand all %f with the quoted path */
646 strbuf_expand(&cmd, params->cmd, strbuf_expand_dict_cb, &dict);
647 strbuf_release(&path);
648
649 strvec_push(&child_process.args, cmd.buf);
650 child_process.use_shell = 1;
651 child_process.in = -1;
652 child_process.out = out;
653
654 if (start_command(&child_process)) {
655 strbuf_release(&cmd);
656 return error(_("cannot fork to run external filter '%s'"),
657 params->cmd);
658 }
659
660 sigchain_push(SIGPIPE, SIG_IGN);
661
662 if (params->src) {
663 write_err = (write_in_full(child_process.in,
664 params->src, params->size) < 0);
665 if (errno == EPIPE)
666 write_err = 0;
667 } else {
668 write_err = copy_fd(params->fd, child_process.in);
669 if (write_err == COPY_WRITE_ERROR && errno == EPIPE)
670 write_err = 0;
671 }
672
673 if (close(child_process.in))
674 write_err = 1;
675 if (write_err)
676 error(_("cannot feed the input to external filter '%s'"),
677 params->cmd);
678
679 sigchain_pop(SIGPIPE);
680
681 status = finish_command(&child_process);
682 if (status)
683 error(_("external filter '%s' failed %d"), params->cmd, status);
684
685 strbuf_release(&cmd);
686 return (write_err || status);
687 }
688
689 static int apply_single_file_filter(const char *path, const char *src, size_t len, int fd,
690 struct strbuf *dst, const char *cmd)
691 {
692 /*
693 * Create a pipeline to have the command filter the buffer's
694 * contents.
695 *
696 * (child --> cmd) --> us
697 */
698 int err = 0;
699 struct strbuf nbuf = STRBUF_INIT;
700 struct async async;
701 struct filter_params params;
702
703 memset(&async, 0, sizeof(async));
704 async.proc = filter_buffer_or_fd;
705 async.data = &params;
706 async.out = -1;
707 params.src = src;
708 params.size = len;
709 params.fd = fd;
710 params.cmd = cmd;
711 params.path = path;
712
713 fflush(NULL);
714 if (start_async(&async))
715 return 0; /* error was already reported */
716
717 if (strbuf_read(&nbuf, async.out, 0) < 0) {
718 err = error(_("read from external filter '%s' failed"), cmd);
719 }
720 if (close(async.out)) {
721 err = error(_("read from external filter '%s' failed"), cmd);
722 }
723 if (finish_async(&async)) {
724 err = error(_("external filter '%s' failed"), cmd);
725 }
726
727 if (!err) {
728 strbuf_swap(dst, &nbuf);
729 }
730 strbuf_release(&nbuf);
731 return !err;
732 }
733
734 #define CAP_CLEAN (1u<<0)
735 #define CAP_SMUDGE (1u<<1)
736 #define CAP_DELAY (1u<<2)
737
738 struct cmd2process {
739 struct subprocess_entry subprocess; /* must be the first member! */
740 unsigned int supported_capabilities;
741 };
742
743 static int subprocess_map_initialized;
744 static struct hashmap subprocess_map;
745
746 static int start_multi_file_filter_fn(struct subprocess_entry *subprocess)
747 {
748 static int versions[] = {2, 0};
749 static struct subprocess_capability capabilities[] = {
750 { "clean", CAP_CLEAN },
751 { "smudge", CAP_SMUDGE },
752 { "delay", CAP_DELAY },
753 { NULL, 0 }
754 };
755 struct cmd2process *entry = (struct cmd2process *)subprocess;
756 return subprocess_handshake(subprocess, "git-filter", versions, NULL,
757 capabilities,
758 &entry->supported_capabilities);
759 }
760
761 static void handle_filter_error(const struct strbuf *filter_status,
762 struct cmd2process *entry,
763 const unsigned int wanted_capability)
764 {
765 if (!strcmp(filter_status->buf, "error"))
766 ; /* The filter signaled a problem with the file. */
767 else if (!strcmp(filter_status->buf, "abort") && wanted_capability) {
768 /*
769 * The filter signaled a permanent problem. Don't try to filter
770 * files with the same command for the lifetime of the current
771 * Git process.
772 */
773 entry->supported_capabilities &= ~wanted_capability;
774 } else {
775 /*
776 * Something went wrong with the protocol filter.
777 * Force shutdown and restart if another blob requires filtering.
778 */
779 error(_("external filter '%s' failed"), entry->subprocess.cmd);
780 subprocess_stop(&subprocess_map, &entry->subprocess);
781 free(entry);
782 }
783 }
784
785 static int apply_multi_file_filter(const char *path, const char *src, size_t len,
786 int fd, struct strbuf *dst, const char *cmd,
787 const unsigned int wanted_capability,
788 const struct checkout_metadata *meta,
789 struct delayed_checkout *dco)
790 {
791 int err;
792 int can_delay = 0;
793 struct cmd2process *entry;
794 struct child_process *process;
795 struct strbuf nbuf = STRBUF_INIT;
796 struct strbuf filter_status = STRBUF_INIT;
797 const char *filter_type;
798
799 if (!subprocess_map_initialized) {
800 subprocess_map_initialized = 1;
801 hashmap_init(&subprocess_map, cmd2process_cmp, NULL, 0);
802 entry = NULL;
803 } else {
804 entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
805 }
806
807 fflush(NULL);
808
809 if (!entry) {
810 entry = xmalloc(sizeof(*entry));
811 entry->supported_capabilities = 0;
812
813 if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_multi_file_filter_fn)) {
814 free(entry);
815 return 0;
816 }
817 }
818 process = &entry->subprocess.process;
819
820 if (!(entry->supported_capabilities & wanted_capability))
821 return 0;
822
823 if (wanted_capability & CAP_CLEAN)
824 filter_type = "clean";
825 else if (wanted_capability & CAP_SMUDGE)
826 filter_type = "smudge";
827 else
828 die(_("unexpected filter type"));
829
830 sigchain_push(SIGPIPE, SIG_IGN);
831
832 assert(strlen(filter_type) < LARGE_PACKET_DATA_MAX - strlen("command=\n"));
833 err = packet_write_fmt_gently(process->in, "command=%s\n", filter_type);
834 if (err)
835 goto done;
836
837 err = strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n");
838 if (err) {
839 error(_("path name too long for external filter"));
840 goto done;
841 }
842
843 err = packet_write_fmt_gently(process->in, "pathname=%s\n", path);
844 if (err)
845 goto done;
846
847 if (meta && meta->refname) {
848 err = packet_write_fmt_gently(process->in, "ref=%s\n", meta->refname);
849 if (err)
850 goto done;
851 }
852
853 if (meta && !is_null_oid(&meta->treeish)) {
854 err = packet_write_fmt_gently(process->in, "treeish=%s\n", oid_to_hex(&meta->treeish));
855 if (err)
856 goto done;
857 }
858
859 if (meta && !is_null_oid(&meta->blob)) {
860 err = packet_write_fmt_gently(process->in, "blob=%s\n", oid_to_hex(&meta->blob));
861 if (err)
862 goto done;
863 }
864
865 if ((entry->supported_capabilities & CAP_DELAY) &&
866 dco && dco->state == CE_CAN_DELAY) {
867 can_delay = 1;
868 err = packet_write_fmt_gently(process->in, "can-delay=1\n");
869 if (err)
870 goto done;
871 }
872
873 err = packet_flush_gently(process->in);
874 if (err)
875 goto done;
876
877 if (fd >= 0)
878 err = write_packetized_from_fd_no_flush(fd, process->in);
879 else
880 err = write_packetized_from_buf_no_flush(src, len, process->in);
881 if (err)
882 goto done;
883
884 err = packet_flush_gently(process->in);
885 if (err)
886 goto done;
887
888 err = subprocess_read_status(process->out, &filter_status);
889 if (err)
890 goto done;
891
892 if (can_delay && !strcmp(filter_status.buf, "delayed")) {
893 string_list_insert(&dco->filters, cmd);
894 string_list_insert(&dco->paths, path);
895 } else {
896 /* The filter got the blob and wants to send us a response. */
897 err = strcmp(filter_status.buf, "success");
898 if (err)
899 goto done;
900
901 err = read_packetized_to_strbuf(process->out, &nbuf,
902 PACKET_READ_GENTLE_ON_EOF) < 0;
903 if (err)
904 goto done;
905
906 err = subprocess_read_status(process->out, &filter_status);
907 if (err)
908 goto done;
909
910 err = strcmp(filter_status.buf, "success");
911 }
912
913 done:
914 sigchain_pop(SIGPIPE);
915
916 if (err)
917 handle_filter_error(&filter_status, entry, wanted_capability);
918 else
919 strbuf_swap(dst, &nbuf);
920 strbuf_release(&nbuf);
921 strbuf_release(&filter_status);
922 return !err;
923 }
924
925
926 int async_query_available_blobs(const char *cmd, struct string_list *available_paths)
927 {
928 int err;
929 char *line;
930 struct cmd2process *entry;
931 struct child_process *process;
932 struct strbuf filter_status = STRBUF_INIT;
933
934 assert(subprocess_map_initialized);
935 entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
936 if (!entry) {
937 error(_("external filter '%s' is not available anymore although "
938 "not all paths have been filtered"), cmd);
939 return 0;
940 }
941 process = &entry->subprocess.process;
942 sigchain_push(SIGPIPE, SIG_IGN);
943
944 err = packet_write_fmt_gently(
945 process->in, "command=list_available_blobs\n");
946 if (err)
947 goto done;
948
949 err = packet_flush_gently(process->in);
950 if (err)
951 goto done;
952
953 while ((line = packet_read_line(process->out, NULL))) {
954 const char *path;
955 if (skip_prefix(line, "pathname=", &path))
956 string_list_insert(available_paths, xstrdup(path));
957 else
958 ; /* ignore unknown keys */
959 }
960
961 err = subprocess_read_status(process->out, &filter_status);
962 if (err)
963 goto done;
964
965 err = strcmp(filter_status.buf, "success");
966
967 done:
968 sigchain_pop(SIGPIPE);
969
970 if (err)
971 handle_filter_error(&filter_status, entry, 0);
972 strbuf_release(&filter_status);
973 return !err;
974 }
975
976 static struct convert_driver {
977 const char *name;
978 struct convert_driver *next;
979 const char *smudge;
980 const char *clean;
981 const char *process;
982 int required;
983 } *user_convert, **user_convert_tail;
984
985 static int apply_filter(const char *path, const char *src, size_t len,
986 int fd, struct strbuf *dst, struct convert_driver *drv,
987 const unsigned int wanted_capability,
988 const struct checkout_metadata *meta,
989 struct delayed_checkout *dco)
990 {
991 const char *cmd = NULL;
992
993 if (!drv)
994 return 0;
995
996 if (!dst)
997 return 1;
998
999 if ((wanted_capability & CAP_CLEAN) && !drv->process && drv->clean)
1000 cmd = drv->clean;
1001 else if ((wanted_capability & CAP_SMUDGE) && !drv->process && drv->smudge)
1002 cmd = drv->smudge;
1003
1004 if (cmd && *cmd)
1005 return apply_single_file_filter(path, src, len, fd, dst, cmd);
1006 else if (drv->process && *drv->process)
1007 return apply_multi_file_filter(path, src, len, fd, dst,
1008 drv->process, wanted_capability, meta, dco);
1009
1010 return 0;
1011 }
1012
1013 static int read_convert_config(const char *var, const char *value, void *cb UNUSED)
1014 {
1015 const char *key, *name;
1016 size_t namelen;
1017 struct convert_driver *drv;
1018
1019 /*
1020 * External conversion drivers are configured using
1021 * "filter.<name>.variable".
1022 */
1023 if (parse_config_key(var, "filter", &name, &namelen, &key) < 0 || !name)
1024 return 0;
1025 for (drv = user_convert; drv; drv = drv->next)
1026 if (!strncmp(drv->name, name, namelen) && !drv->name[namelen])
1027 break;
1028 if (!drv) {
1029 CALLOC_ARRAY(drv, 1);
1030 drv->name = xmemdupz(name, namelen);
1031 *user_convert_tail = drv;
1032 user_convert_tail = &(drv->next);
1033 }
1034
1035 /*
1036 * filter.<name>.smudge and filter.<name>.clean specifies
1037 * the command line:
1038 *
1039 * command-line
1040 *
1041 * The command-line will not be interpolated in any way.
1042 */
1043
1044 if (!strcmp("smudge", key))
1045 return git_config_string(&drv->smudge, var, value);
1046
1047 if (!strcmp("clean", key))
1048 return git_config_string(&drv->clean, var, value);
1049
1050 if (!strcmp("process", key))
1051 return git_config_string(&drv->process, var, value);
1052
1053 if (!strcmp("required", key)) {
1054 drv->required = git_config_bool(var, value);
1055 return 0;
1056 }
1057
1058 return 0;
1059 }
1060
1061 static int count_ident(const char *cp, unsigned long size)
1062 {
1063 /*
1064 * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$"
1065 */
1066 int cnt = 0;
1067 char ch;
1068
1069 while (size) {
1070 ch = *cp++;
1071 size--;
1072 if (ch != '$')
1073 continue;
1074 if (size < 3)
1075 break;
1076 if (memcmp("Id", cp, 2))
1077 continue;
1078 ch = cp[2];
1079 cp += 3;
1080 size -= 3;
1081 if (ch == '$')
1082 cnt++; /* $Id$ */
1083 if (ch != ':')
1084 continue;
1085
1086 /*
1087 * "$Id: ... "; scan up to the closing dollar sign and discard.
1088 */
1089 while (size) {
1090 ch = *cp++;
1091 size--;
1092 if (ch == '$') {
1093 cnt++;
1094 break;
1095 }
1096 if (ch == '\n')
1097 break;
1098 }
1099 }
1100 return cnt;
1101 }
1102
1103 static int ident_to_git(const char *src, size_t len,
1104 struct strbuf *buf, int ident)
1105 {
1106 char *dst, *dollar;
1107
1108 if (!ident || (src && !count_ident(src, len)))
1109 return 0;
1110
1111 if (!buf)
1112 return 1;
1113
1114 /* only grow if not in place */
1115 if (strbuf_avail(buf) + buf->len < len)
1116 strbuf_grow(buf, len - buf->len);
1117 dst = buf->buf;
1118 for (;;) {
1119 dollar = memchr(src, '$', len);
1120 if (!dollar)
1121 break;
1122 memmove(dst, src, dollar + 1 - src);
1123 dst += dollar + 1 - src;
1124 len -= dollar + 1 - src;
1125 src = dollar + 1;
1126
1127 if (len > 3 && !memcmp(src, "Id:", 3)) {
1128 dollar = memchr(src + 3, '$', len - 3);
1129 if (!dollar)
1130 break;
1131 if (memchr(src + 3, '\n', dollar - src - 3)) {
1132 /* Line break before the next dollar. */
1133 continue;
1134 }
1135
1136 memcpy(dst, "Id$", 3);
1137 dst += 3;
1138 len -= dollar + 1 - src;
1139 src = dollar + 1;
1140 }
1141 }
1142 memmove(dst, src, len);
1143 strbuf_setlen(buf, dst + len - buf->buf);
1144 return 1;
1145 }
1146
1147 static int ident_to_worktree(const char *src, size_t len,
1148 struct strbuf *buf, int ident)
1149 {
1150 struct object_id oid;
1151 char *to_free = NULL, *dollar, *spc;
1152 int cnt;
1153
1154 if (!ident)
1155 return 0;
1156
1157 cnt = count_ident(src, len);
1158 if (!cnt)
1159 return 0;
1160
1161 /* are we "faking" in place editing ? */
1162 if (src == buf->buf)
1163 to_free = strbuf_detach(buf, NULL);
1164 hash_object_file(the_hash_algo, src, len, OBJ_BLOB, &oid);
1165
1166 strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + 3));
1167 for (;;) {
1168 /* step 1: run to the next '$' */
1169 dollar = memchr(src, '$', len);
1170 if (!dollar)
1171 break;
1172 strbuf_add(buf, src, dollar + 1 - src);
1173 len -= dollar + 1 - src;
1174 src = dollar + 1;
1175
1176 /* step 2: does it looks like a bit like Id:xxx$ or Id$ ? */
1177 if (len < 3 || memcmp("Id", src, 2))
1178 continue;
1179
1180 /* step 3: skip over Id$ or Id:xxxxx$ */
1181 if (src[2] == '$') {
1182 src += 3;
1183 len -= 3;
1184 } else if (src[2] == ':') {
1185 /*
1186 * It's possible that an expanded Id has crept its way into the
1187 * repository, we cope with that by stripping the expansion out.
1188 * This is probably not a good idea, since it will cause changes
1189 * on checkout, which won't go away by stash, but let's keep it
1190 * for git-style ids.
1191 */
1192 dollar = memchr(src + 3, '$', len - 3);
1193 if (!dollar) {
1194 /* incomplete keyword, no more '$', so just quit the loop */
1195 break;
1196 }
1197
1198 if (memchr(src + 3, '\n', dollar - src - 3)) {
1199 /* Line break before the next dollar. */
1200 continue;
1201 }
1202
1203 spc = memchr(src + 4, ' ', dollar - src - 4);
1204 if (spc && spc < dollar-1) {
1205 /* There are spaces in unexpected places.
1206 * This is probably an id from some other
1207 * versioning system. Keep it for now.
1208 */
1209 continue;
1210 }
1211
1212 len -= dollar + 1 - src;
1213 src = dollar + 1;
1214 } else {
1215 /* it wasn't a "Id$" or "Id:xxxx$" */
1216 continue;
1217 }
1218
1219 /* step 4: substitute */
1220 strbuf_addstr(buf, "Id: ");
1221 strbuf_addstr(buf, oid_to_hex(&oid));
1222 strbuf_addstr(buf, " $");
1223 }
1224 strbuf_add(buf, src, len);
1225
1226 free(to_free);
1227 return 1;
1228 }
1229
1230 static const char *git_path_check_encoding(struct attr_check_item *check)
1231 {
1232 const char *value = check->value;
1233
1234 if (ATTR_UNSET(value) || !strlen(value))
1235 return NULL;
1236
1237 if (ATTR_TRUE(value) || ATTR_FALSE(value)) {
1238 die(_("true/false are no valid working-tree-encodings"));
1239 }
1240
1241 /* Don't encode to the default encoding */
1242 if (same_encoding(value, default_encoding))
1243 return NULL;
1244
1245 return value;
1246 }
1247
1248 static enum convert_crlf_action git_path_check_crlf(struct attr_check_item *check)
1249 {
1250 const char *value = check->value;
1251
1252 if (ATTR_TRUE(value))
1253 return CRLF_TEXT;
1254 else if (ATTR_FALSE(value))
1255 return CRLF_BINARY;
1256 else if (ATTR_UNSET(value))
1257 ;
1258 else if (!strcmp(value, "input"))
1259 return CRLF_TEXT_INPUT;
1260 else if (!strcmp(value, "auto"))
1261 return CRLF_AUTO;
1262 return CRLF_UNDEFINED;
1263 }
1264
1265 static enum eol git_path_check_eol(struct attr_check_item *check)
1266 {
1267 const char *value = check->value;
1268
1269 if (ATTR_UNSET(value))
1270 ;
1271 else if (!strcmp(value, "lf"))
1272 return EOL_LF;
1273 else if (!strcmp(value, "crlf"))
1274 return EOL_CRLF;
1275 return EOL_UNSET;
1276 }
1277
1278 static struct convert_driver *git_path_check_convert(struct attr_check_item *check)
1279 {
1280 const char *value = check->value;
1281 struct convert_driver *drv;
1282
1283 if (ATTR_TRUE(value) || ATTR_FALSE(value) || ATTR_UNSET(value))
1284 return NULL;
1285 for (drv = user_convert; drv; drv = drv->next)
1286 if (!strcmp(value, drv->name))
1287 return drv;
1288 return NULL;
1289 }
1290
1291 static int git_path_check_ident(struct attr_check_item *check)
1292 {
1293 const char *value = check->value;
1294
1295 return !!ATTR_TRUE(value);
1296 }
1297
1298 static struct attr_check *check;
1299
1300 void convert_attrs(struct index_state *istate,
1301 struct conv_attrs *ca, const char *path)
1302 {
1303 struct attr_check_item *ccheck = NULL;
1304
1305 if (!check) {
1306 check = attr_check_initl("crlf", "ident", "filter",
1307 "eol", "text", "working-tree-encoding",
1308 NULL);
1309 user_convert_tail = &user_convert;
1310 git_config(read_convert_config, NULL);
1311 }
1312
1313 git_check_attr(istate, NULL, path, check);
1314 ccheck = check->items;
1315 ca->crlf_action = git_path_check_crlf(ccheck + 4);
1316 if (ca->crlf_action == CRLF_UNDEFINED)
1317 ca->crlf_action = git_path_check_crlf(ccheck + 0);
1318 ca->ident = git_path_check_ident(ccheck + 1);
1319 ca->drv = git_path_check_convert(ccheck + 2);
1320 if (ca->crlf_action != CRLF_BINARY) {
1321 enum eol eol_attr = git_path_check_eol(ccheck + 3);
1322 if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_LF)
1323 ca->crlf_action = CRLF_AUTO_INPUT;
1324 else if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_CRLF)
1325 ca->crlf_action = CRLF_AUTO_CRLF;
1326 else if (eol_attr == EOL_LF)
1327 ca->crlf_action = CRLF_TEXT_INPUT;
1328 else if (eol_attr == EOL_CRLF)
1329 ca->crlf_action = CRLF_TEXT_CRLF;
1330 }
1331 ca->working_tree_encoding = git_path_check_encoding(ccheck + 5);
1332
1333 /* Save attr and make a decision for action */
1334 ca->attr_action = ca->crlf_action;
1335 if (ca->crlf_action == CRLF_TEXT)
1336 ca->crlf_action = text_eol_is_crlf() ? CRLF_TEXT_CRLF : CRLF_TEXT_INPUT;
1337 if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_FALSE)
1338 ca->crlf_action = CRLF_BINARY;
1339 if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_TRUE)
1340 ca->crlf_action = CRLF_AUTO_CRLF;
1341 if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_INPUT)
1342 ca->crlf_action = CRLF_AUTO_INPUT;
1343 }
1344
1345 void reset_parsed_attributes(void)
1346 {
1347 struct convert_driver *drv, *next;
1348
1349 attr_check_free(check);
1350 check = NULL;
1351 reset_merge_attributes();
1352
1353 for (drv = user_convert; drv; drv = next) {
1354 next = drv->next;
1355 free((void *)drv->name);
1356 free(drv);
1357 }
1358 user_convert = NULL;
1359 user_convert_tail = NULL;
1360 }
1361
1362 int would_convert_to_git_filter_fd(struct index_state *istate, const char *path)
1363 {
1364 struct conv_attrs ca;
1365
1366 convert_attrs(istate, &ca, path);
1367 if (!ca.drv)
1368 return 0;
1369
1370 /*
1371 * Apply a filter to an fd only if the filter is required to succeed.
1372 * We must die if the filter fails, because the original data before
1373 * filtering is not available.
1374 */
1375 if (!ca.drv->required)
1376 return 0;
1377
1378 return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN, NULL, NULL);
1379 }
1380
1381 const char *get_convert_attr_ascii(struct index_state *istate, const char *path)
1382 {
1383 struct conv_attrs ca;
1384
1385 convert_attrs(istate, &ca, path);
1386 switch (ca.attr_action) {
1387 case CRLF_UNDEFINED:
1388 return "";
1389 case CRLF_BINARY:
1390 return "-text";
1391 case CRLF_TEXT:
1392 return "text";
1393 case CRLF_TEXT_INPUT:
1394 return "text eol=lf";
1395 case CRLF_TEXT_CRLF:
1396 return "text eol=crlf";
1397 case CRLF_AUTO:
1398 return "text=auto";
1399 case CRLF_AUTO_CRLF:
1400 return "text=auto eol=crlf";
1401 case CRLF_AUTO_INPUT:
1402 return "text=auto eol=lf";
1403 }
1404 return "";
1405 }
1406
1407 int convert_to_git(struct index_state *istate,
1408 const char *path, const char *src, size_t len,
1409 struct strbuf *dst, int conv_flags)
1410 {
1411 int ret = 0;
1412 struct conv_attrs ca;
1413
1414 convert_attrs(istate, &ca, path);
1415
1416 ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN, NULL, NULL);
1417 if (!ret && ca.drv && ca.drv->required)
1418 die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1419
1420 if (ret && dst) {
1421 src = dst->buf;
1422 len = dst->len;
1423 }
1424
1425 ret |= encode_to_git(path, src, len, dst, ca.working_tree_encoding, conv_flags);
1426 if (ret && dst) {
1427 src = dst->buf;
1428 len = dst->len;
1429 }
1430
1431 if (!(conv_flags & CONV_EOL_KEEP_CRLF)) {
1432 ret |= crlf_to_git(istate, path, src, len, dst, ca.crlf_action, conv_flags);
1433 if (ret && dst) {
1434 src = dst->buf;
1435 len = dst->len;
1436 }
1437 }
1438 return ret | ident_to_git(src, len, dst, ca.ident);
1439 }
1440
1441 void convert_to_git_filter_fd(struct index_state *istate,
1442 const char *path, int fd, struct strbuf *dst,
1443 int conv_flags)
1444 {
1445 struct conv_attrs ca;
1446 convert_attrs(istate, &ca, path);
1447
1448 assert(ca.drv);
1449
1450 if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN, NULL, NULL))
1451 die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1452
1453 encode_to_git(path, dst->buf, dst->len, dst, ca.working_tree_encoding, conv_flags);
1454 crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, conv_flags);
1455 ident_to_git(dst->buf, dst->len, dst, ca.ident);
1456 }
1457
1458 static int convert_to_working_tree_ca_internal(const struct conv_attrs *ca,
1459 const char *path, const char *src,
1460 size_t len, struct strbuf *dst,
1461 int normalizing,
1462 const struct checkout_metadata *meta,
1463 struct delayed_checkout *dco)
1464 {
1465 int ret = 0, ret_filter = 0;
1466
1467 ret |= ident_to_worktree(src, len, dst, ca->ident);
1468 if (ret) {
1469 src = dst->buf;
1470 len = dst->len;
1471 }
1472 /*
1473 * CRLF conversion can be skipped if normalizing, unless there
1474 * is a smudge or process filter (even if the process filter doesn't
1475 * support smudge). The filters might expect CRLFs.
1476 */
1477 if ((ca->drv && (ca->drv->smudge || ca->drv->process)) || !normalizing) {
1478 ret |= crlf_to_worktree(src, len, dst, ca->crlf_action);
1479 if (ret) {
1480 src = dst->buf;
1481 len = dst->len;
1482 }
1483 }
1484
1485 ret |= encode_to_worktree(path, src, len, dst, ca->working_tree_encoding);
1486 if (ret) {
1487 src = dst->buf;
1488 len = dst->len;
1489 }
1490
1491 ret_filter = apply_filter(
1492 path, src, len, -1, dst, ca->drv, CAP_SMUDGE, meta, dco);
1493 if (!ret_filter && ca->drv && ca->drv->required)
1494 die(_("%s: smudge filter %s failed"), path, ca->drv->name);
1495
1496 return ret | ret_filter;
1497 }
1498
1499 int async_convert_to_working_tree_ca(const struct conv_attrs *ca,
1500 const char *path, const char *src,
1501 size_t len, struct strbuf *dst,
1502 const struct checkout_metadata *meta,
1503 void *dco)
1504 {
1505 return convert_to_working_tree_ca_internal(ca, path, src, len, dst, 0,
1506 meta, dco);
1507 }
1508
1509 int convert_to_working_tree_ca(const struct conv_attrs *ca,
1510 const char *path, const char *src,
1511 size_t len, struct strbuf *dst,
1512 const struct checkout_metadata *meta)
1513 {
1514 return convert_to_working_tree_ca_internal(ca, path, src, len, dst, 0,
1515 meta, NULL);
1516 }
1517
1518 int renormalize_buffer(struct index_state *istate, const char *path,
1519 const char *src, size_t len, struct strbuf *dst)
1520 {
1521 struct conv_attrs ca;
1522 int ret;
1523
1524 convert_attrs(istate, &ca, path);
1525 ret = convert_to_working_tree_ca_internal(&ca, path, src, len, dst, 1,
1526 NULL, NULL);
1527 if (ret) {
1528 src = dst->buf;
1529 len = dst->len;
1530 }
1531 return ret | convert_to_git(istate, path, src, len, dst, CONV_EOL_RENORMALIZE);
1532 }
1533
1534 /*****************************************************************
1535 *
1536 * Streaming conversion support
1537 *
1538 *****************************************************************/
1539
1540 typedef int (*filter_fn)(struct stream_filter *,
1541 const char *input, size_t *isize_p,
1542 char *output, size_t *osize_p);
1543 typedef void (*free_fn)(struct stream_filter *);
1544
1545 struct stream_filter_vtbl {
1546 filter_fn filter;
1547 free_fn free;
1548 };
1549
1550 struct stream_filter {
1551 struct stream_filter_vtbl *vtbl;
1552 };
1553
1554 static int null_filter_fn(struct stream_filter *filter UNUSED,
1555 const char *input, size_t *isize_p,
1556 char *output, size_t *osize_p)
1557 {
1558 size_t count;
1559
1560 if (!input)
1561 return 0; /* we do not keep any states */
1562 count = *isize_p;
1563 if (*osize_p < count)
1564 count = *osize_p;
1565 if (count) {
1566 memmove(output, input, count);
1567 *isize_p -= count;
1568 *osize_p -= count;
1569 }
1570 return 0;
1571 }
1572
1573 static void null_free_fn(struct stream_filter *filter UNUSED)
1574 {
1575 ; /* nothing -- null instances are shared */
1576 }
1577
1578 static struct stream_filter_vtbl null_vtbl = {
1579 .filter = null_filter_fn,
1580 .free = null_free_fn,
1581 };
1582
1583 static struct stream_filter null_filter_singleton = {
1584 .vtbl = &null_vtbl,
1585 };
1586
1587 int is_null_stream_filter(struct stream_filter *filter)
1588 {
1589 return filter == &null_filter_singleton;
1590 }
1591
1592
1593 /*
1594 * LF-to-CRLF filter
1595 */
1596
1597 struct lf_to_crlf_filter {
1598 struct stream_filter filter;
1599 unsigned has_held:1;
1600 char held;
1601 };
1602
1603 static int lf_to_crlf_filter_fn(struct stream_filter *filter,
1604 const char *input, size_t *isize_p,
1605 char *output, size_t *osize_p)
1606 {
1607 size_t count, o = 0;
1608 struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter;
1609
1610 /*
1611 * We may be holding onto the CR to see if it is followed by a
1612 * LF, in which case we would need to go to the main loop.
1613 * Otherwise, just emit it to the output stream.
1614 */
1615 if (lf_to_crlf->has_held && (lf_to_crlf->held != '\r' || !input)) {
1616 output[o++] = lf_to_crlf->held;
1617 lf_to_crlf->has_held = 0;
1618 }
1619
1620 /* We are told to drain */
1621 if (!input) {
1622 *osize_p -= o;
1623 return 0;
1624 }
1625
1626 count = *isize_p;
1627 if (count || lf_to_crlf->has_held) {
1628 size_t i;
1629 int was_cr = 0;
1630
1631 if (lf_to_crlf->has_held) {
1632 was_cr = 1;
1633 lf_to_crlf->has_held = 0;
1634 }
1635
1636 for (i = 0; o < *osize_p && i < count; i++) {
1637 char ch = input[i];
1638
1639 if (ch == '\n') {
1640 output[o++] = '\r';
1641 } else if (was_cr) {
1642 /*
1643 * Previous round saw CR and it is not followed
1644 * by a LF; emit the CR before processing the
1645 * current character.
1646 */
1647 output[o++] = '\r';
1648 }
1649
1650 /*
1651 * We may have consumed the last output slot,
1652 * in which case we need to break out of this
1653 * loop; hold the current character before
1654 * returning.
1655 */
1656 if (*osize_p <= o) {
1657 lf_to_crlf->has_held = 1;
1658 lf_to_crlf->held = ch;
1659 continue; /* break but increment i */
1660 }
1661
1662 if (ch == '\r') {
1663 was_cr = 1;
1664 continue;
1665 }
1666
1667 was_cr = 0;
1668 output[o++] = ch;
1669 }
1670
1671 *osize_p -= o;
1672 *isize_p -= i;
1673
1674 if (!lf_to_crlf->has_held && was_cr) {
1675 lf_to_crlf->has_held = 1;
1676 lf_to_crlf->held = '\r';
1677 }
1678 }
1679 return 0;
1680 }
1681
1682 static void lf_to_crlf_free_fn(struct stream_filter *filter)
1683 {
1684 free(filter);
1685 }
1686
1687 static struct stream_filter_vtbl lf_to_crlf_vtbl = {
1688 .filter = lf_to_crlf_filter_fn,
1689 .free = lf_to_crlf_free_fn,
1690 };
1691
1692 static struct stream_filter *lf_to_crlf_filter(void)
1693 {
1694 struct lf_to_crlf_filter *lf_to_crlf = xcalloc(1, sizeof(*lf_to_crlf));
1695
1696 lf_to_crlf->filter.vtbl = &lf_to_crlf_vtbl;
1697 return (struct stream_filter *)lf_to_crlf;
1698 }
1699
1700 /*
1701 * Cascade filter
1702 */
1703 #define FILTER_BUFFER 1024
1704 struct cascade_filter {
1705 struct stream_filter filter;
1706 struct stream_filter *one;
1707 struct stream_filter *two;
1708 char buf[FILTER_BUFFER];
1709 int end, ptr;
1710 };
1711
1712 static int cascade_filter_fn(struct stream_filter *filter,
1713 const char *input, size_t *isize_p,
1714 char *output, size_t *osize_p)
1715 {
1716 struct cascade_filter *cas = (struct cascade_filter *) filter;
1717 size_t filled = 0;
1718 size_t sz = *osize_p;
1719 size_t to_feed, remaining;
1720
1721 /*
1722 * input -- (one) --> buf -- (two) --> output
1723 */
1724 while (filled < sz) {
1725 remaining = sz - filled;
1726
1727 /* do we already have something to feed two with? */
1728 if (cas->ptr < cas->end) {
1729 to_feed = cas->end - cas->ptr;
1730 if (stream_filter(cas->two,
1731 cas->buf + cas->ptr, &to_feed,
1732 output + filled, &remaining))
1733 return -1;
1734 cas->ptr += (cas->end - cas->ptr) - to_feed;
1735 filled = sz - remaining;
1736 continue;
1737 }
1738
1739 /* feed one from upstream and have it emit into our buffer */
1740 to_feed = input ? *isize_p : 0;
1741 if (input && !to_feed)
1742 break;
1743 remaining = sizeof(cas->buf);
1744 if (stream_filter(cas->one,
1745 input, &to_feed,
1746 cas->buf, &remaining))
1747 return -1;
1748 cas->end = sizeof(cas->buf) - remaining;
1749 cas->ptr = 0;
1750 if (input) {
1751 size_t fed = *isize_p - to_feed;
1752 *isize_p -= fed;
1753 input += fed;
1754 }
1755
1756 /* do we know that we drained one completely? */
1757 if (input || cas->end)
1758 continue;
1759
1760 /* tell two to drain; we have nothing more to give it */
1761 to_feed = 0;
1762 remaining = sz - filled;
1763 if (stream_filter(cas->two,
1764 NULL, &to_feed,
1765 output + filled, &remaining))
1766 return -1;
1767 if (remaining == (sz - filled))
1768 break; /* completely drained two */
1769 filled = sz - remaining;
1770 }
1771 *osize_p -= filled;
1772 return 0;
1773 }
1774
1775 static void cascade_free_fn(struct stream_filter *filter)
1776 {
1777 struct cascade_filter *cas = (struct cascade_filter *)filter;
1778 free_stream_filter(cas->one);
1779 free_stream_filter(cas->two);
1780 free(filter);
1781 }
1782
1783 static struct stream_filter_vtbl cascade_vtbl = {
1784 .filter = cascade_filter_fn,
1785 .free = cascade_free_fn,
1786 };
1787
1788 static struct stream_filter *cascade_filter(struct stream_filter *one,
1789 struct stream_filter *two)
1790 {
1791 struct cascade_filter *cascade;
1792
1793 if (!one || is_null_stream_filter(one))
1794 return two;
1795 if (!two || is_null_stream_filter(two))
1796 return one;
1797
1798 cascade = xmalloc(sizeof(*cascade));
1799 cascade->one = one;
1800 cascade->two = two;
1801 cascade->end = cascade->ptr = 0;
1802 cascade->filter.vtbl = &cascade_vtbl;
1803 return (struct stream_filter *)cascade;
1804 }
1805
1806 /*
1807 * ident filter
1808 */
1809 #define IDENT_DRAINING (-1)
1810 #define IDENT_SKIPPING (-2)
1811 struct ident_filter {
1812 struct stream_filter filter;
1813 struct strbuf left;
1814 int state;
1815 char ident[GIT_MAX_HEXSZ + 5]; /* ": x40 $" */
1816 };
1817
1818 static int is_foreign_ident(const char *str)
1819 {
1820 int i;
1821
1822 if (!skip_prefix(str, "$Id: ", &str))
1823 return 0;
1824 for (i = 0; str[i]; i++) {
1825 if (isspace(str[i]) && str[i+1] != '$')
1826 return 1;
1827 }
1828 return 0;
1829 }
1830
1831 static void ident_drain(struct ident_filter *ident, char **output_p, size_t *osize_p)
1832 {
1833 size_t to_drain = ident->left.len;
1834
1835 if (*osize_p < to_drain)
1836 to_drain = *osize_p;
1837 if (to_drain) {
1838 memcpy(*output_p, ident->left.buf, to_drain);
1839 strbuf_remove(&ident->left, 0, to_drain);
1840 *output_p += to_drain;
1841 *osize_p -= to_drain;
1842 }
1843 if (!ident->left.len)
1844 ident->state = 0;
1845 }
1846
1847 static int ident_filter_fn(struct stream_filter *filter,
1848 const char *input, size_t *isize_p,
1849 char *output, size_t *osize_p)
1850 {
1851 struct ident_filter *ident = (struct ident_filter *)filter;
1852 static const char head[] = "$Id";
1853
1854 if (!input) {
1855 /* drain upon eof */
1856 switch (ident->state) {
1857 default:
1858 strbuf_add(&ident->left, head, ident->state);
1859 /* fallthrough */
1860 case IDENT_SKIPPING:
1861 /* fallthrough */
1862 case IDENT_DRAINING:
1863 ident_drain(ident, &output, osize_p);
1864 }
1865 return 0;
1866 }
1867
1868 while (*isize_p || (ident->state == IDENT_DRAINING)) {
1869 int ch;
1870
1871 if (ident->state == IDENT_DRAINING) {
1872 ident_drain(ident, &output, osize_p);
1873 if (!*osize_p)
1874 break;
1875 continue;
1876 }
1877
1878 ch = *(input++);
1879 (*isize_p)--;
1880
1881 if (ident->state == IDENT_SKIPPING) {
1882 /*
1883 * Skipping until '$' or LF, but keeping them
1884 * in case it is a foreign ident.
1885 */
1886 strbuf_addch(&ident->left, ch);
1887 if (ch != '\n' && ch != '$')
1888 continue;
1889 if (ch == '$' && !is_foreign_ident(ident->left.buf)) {
1890 strbuf_setlen(&ident->left, sizeof(head) - 1);
1891 strbuf_addstr(&ident->left, ident->ident);
1892 }
1893 ident->state = IDENT_DRAINING;
1894 continue;
1895 }
1896
1897 if (ident->state < sizeof(head) &&
1898 head[ident->state] == ch) {
1899 ident->state++;
1900 continue;
1901 }
1902
1903 if (ident->state)
1904 strbuf_add(&ident->left, head, ident->state);
1905 if (ident->state == sizeof(head) - 1) {
1906 if (ch != ':' && ch != '$') {
1907 strbuf_addch(&ident->left, ch);
1908 ident->state = 0;
1909 continue;
1910 }
1911
1912 if (ch == ':') {
1913 strbuf_addch(&ident->left, ch);
1914 ident->state = IDENT_SKIPPING;
1915 } else {
1916 strbuf_addstr(&ident->left, ident->ident);
1917 ident->state = IDENT_DRAINING;
1918 }
1919 continue;
1920 }
1921
1922 strbuf_addch(&ident->left, ch);
1923 ident->state = IDENT_DRAINING;
1924 }
1925 return 0;
1926 }
1927
1928 static void ident_free_fn(struct stream_filter *filter)
1929 {
1930 struct ident_filter *ident = (struct ident_filter *)filter;
1931 strbuf_release(&ident->left);
1932 free(filter);
1933 }
1934
1935 static struct stream_filter_vtbl ident_vtbl = {
1936 .filter = ident_filter_fn,
1937 .free = ident_free_fn,
1938 };
1939
1940 static struct stream_filter *ident_filter(const struct object_id *oid)
1941 {
1942 struct ident_filter *ident = xmalloc(sizeof(*ident));
1943
1944 xsnprintf(ident->ident, sizeof(ident->ident),
1945 ": %s $", oid_to_hex(oid));
1946 strbuf_init(&ident->left, 0);
1947 ident->filter.vtbl = &ident_vtbl;
1948 ident->state = 0;
1949 return (struct stream_filter *)ident;
1950 }
1951
1952 /*
1953 * Return an appropriately constructed filter for the given ca, or NULL if
1954 * the contents cannot be filtered without reading the whole thing
1955 * in-core.
1956 *
1957 * Note that you would be crazy to set CRLF, smudge/clean or ident to a
1958 * large binary blob you would want us not to slurp into the memory!
1959 */
1960 struct stream_filter *get_stream_filter_ca(const struct conv_attrs *ca,
1961 const struct object_id *oid)
1962 {
1963 struct stream_filter *filter = NULL;
1964
1965 if (classify_conv_attrs(ca) != CA_CLASS_STREAMABLE)
1966 return NULL;
1967
1968 if (ca->ident)
1969 filter = ident_filter(oid);
1970
1971 if (output_eol(ca->crlf_action) == EOL_CRLF)
1972 filter = cascade_filter(filter, lf_to_crlf_filter());
1973 else
1974 filter = cascade_filter(filter, &null_filter_singleton);
1975
1976 return filter;
1977 }
1978
1979 struct stream_filter *get_stream_filter(struct index_state *istate,
1980 const char *path,
1981 const struct object_id *oid)
1982 {
1983 struct conv_attrs ca;
1984 convert_attrs(istate, &ca, path);
1985 return get_stream_filter_ca(&ca, oid);
1986 }
1987
1988 void free_stream_filter(struct stream_filter *filter)
1989 {
1990 filter->vtbl->free(filter);
1991 }
1992
1993 int stream_filter(struct stream_filter *filter,
1994 const char *input, size_t *isize_p,
1995 char *output, size_t *osize_p)
1996 {
1997 return filter->vtbl->filter(filter, input, isize_p, output, osize_p);
1998 }
1999
2000 void init_checkout_metadata(struct checkout_metadata *meta, const char *refname,
2001 const struct object_id *treeish,
2002 const struct object_id *blob)
2003 {
2004 memset(meta, 0, sizeof(*meta));
2005 if (refname)
2006 meta->refname = refname;
2007 if (treeish)
2008 oidcpy(&meta->treeish, treeish);
2009 if (blob)
2010 oidcpy(&meta->blob, blob);
2011 }
2012
2013 void clone_checkout_metadata(struct checkout_metadata *dst,
2014 const struct checkout_metadata *src,
2015 const struct object_id *blob)
2016 {
2017 memcpy(dst, src, sizeof(*dst));
2018 if (blob)
2019 oidcpy(&dst->blob, blob);
2020 }
2021
2022 enum conv_attrs_classification classify_conv_attrs(const struct conv_attrs *ca)
2023 {
2024 if (ca->drv) {
2025 if (ca->drv->process)
2026 return CA_CLASS_INCORE_PROCESS;
2027 if (ca->drv->smudge || ca->drv->clean)
2028 return CA_CLASS_INCORE_FILTER;
2029 }
2030
2031 if (ca->working_tree_encoding)
2032 return CA_CLASS_INCORE;
2033
2034 if (ca->crlf_action == CRLF_AUTO || ca->crlf_action == CRLF_AUTO_CRLF)
2035 return CA_CLASS_INCORE;
2036
2037 return CA_CLASS_STREAMABLE;
2038 }
Unnamed repository; edit this file 'description' to name the repository.