fast-export: do automatic reencoding of commit messages only if requested

Automatic re-encoding of commit messages (and dropping of the encoding
header) hurts attempts to do reversible history rewrites (e.g. sha1sum
<-> sha256sum transitions, some subtree rewrites), and seems
inconsistent with the general principle followed elsewhere in
fast-export of requiring explicit user requests to modify the output
(e.g. --signed-tags=strip, --tag-of-filtered-object=rewrite).  Add a
--reencode flag that the user can use to specify, and like other
fast-export flags, default it to 'abort'.

Signed-off-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-fast-export.txt b/Documentation/git-fast-export.txt
index 64c01ba91884df1ec8e49ddc8fe852f1fb2a9425..11427acdde68e659ca5d443beff037df8c9f3ebb 100644 (file)
--- a/Documentation/git-fast-export.txt
+++ b/Documentation/git-fast-export.txt
@@ -129,6 +129,13 @@ marks the same across runs.
        for intermediary filters (e.g. for rewriting commit messages
        which refer to older commits, or for stripping blobs by id).
 
+--reencode=(yes|no|abort)::
+       Specify how to handle `encoding` header in commit objects.  When
+       asking to 'abort' (which is the default), this program will die
+       when encountering such a commit object.  With 'yes', the commit
+       message will be reencoded into UTF-8.  With 'no', the original
+       encoding will be preserved.
+
 --refspec::
        Apply the specified refspec to each ref exported. Multiple of them can
        be specified.

diff --git a/builtin/fast-export.c b/builtin/fast-export.c
index 66331fa4010c6c2a66feefd7734bba426f3062af..c22cef3b2faff945148029197a32253540c24f73 100644 (file)
--- a/builtin/fast-export.c
+++ b/builtin/fast-export.c
@@ -33,6 +33,7 @@ static const char *fast_export_usage[] = {
 static int progress;
 static enum { SIGNED_TAG_ABORT, VERBATIM, WARN, WARN_STRIP, STRIP } signed_tag_mode = SIGNED_TAG_ABORT;
 static enum { TAG_FILTERING_ABORT, DROP, REWRITE } tag_of_filtered_mode = TAG_FILTERING_ABORT;
+static enum { REENCODE_ABORT, REENCODE_YES, REENCODE_NO } reencode_mode = REENCODE_ABORT;
 static int fake_missing_tagger;
 static int use_done_feature;
 static int no_data;
@@ -77,6 +78,31 @@ static int parse_opt_tag_of_filtered_mode(const struct option *opt,
        return 0;
 }
 
+static int parse_opt_reencode_mode(const struct option *opt,
+                                  const char *arg, int unset)
+{
+       if (unset) {
+               reencode_mode = REENCODE_ABORT;
+               return 0;
+       }
+
+       switch (git_parse_maybe_bool(arg)) {
+       case 0:
+               reencode_mode = REENCODE_NO;
+               break;
+       case 1:
+               reencode_mode = REENCODE_YES;
+               break;
+       default:
+               if (!strcasecmp(arg, "abort"))
+                       reencode_mode = REENCODE_ABORT;
+               else
+                       return error("Unknown reencoding mode: %s", arg);
+       }
+
+       return 0;
+}
+
 static struct decoration idnums;
 static uint32_t last_idnum;
 
@@ -633,10 +659,21 @@ static void handle_commit(struct commit *commit, struct rev_info *rev,
        }
 
        mark_next_object(&commit->object);
-       if (anonymize)
+       if (anonymize) {
                reencoded = anonymize_commit_message(message);
-       else if (!is_encoding_utf8(encoding))
-               reencoded = reencode_string(message, "UTF-8", encoding);
+       } else if (encoding) {
+               switch(reencode_mode) {
+               case REENCODE_YES:
+                       reencoded = reencode_string(message, "UTF-8", encoding);
+                       break;
+               case REENCODE_NO:
+                       break;
+               case REENCODE_ABORT:
+                       die("Encountered commit-specific encoding %s in commit "
+                           "%s; use --reencode=[yes|no] to handle it",
+                           encoding, oid_to_hex(&commit->object.oid));
+               }
+       }
        if (!commit->parents)
                printf("reset %s\n", refname);
        printf("commit %s\nmark :%"PRIu32"\n", refname, last_idnum);
@@ -1091,6 +1128,9 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix)
                OPT_CALLBACK(0, "tag-of-filtered-object", &tag_of_filtered_mode, N_("mode"),
                             N_("select handling of tags that tag filtered objects"),
                             parse_opt_tag_of_filtered_mode),
+               OPT_CALLBACK(0, "reencode", &reencode_mode, N_("mode"),
+                            N_("select handling of commit messages in an alternate encoding"),
+                            parse_opt_reencode_mode),
                OPT_STRING(0, "export-marks", &export_filename, N_("file"),
                             N_("Dump marks to this file")),
                OPT_STRING(0, "import-marks", &import_filename, N_("file"),

diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh
index e2ab8eddc06509a25a25062760ca9ff8cb0000c5..b4004e05c2a72c4daf95c4ce74b2145a9486efc4 100755 (executable)
--- a/t/t9350-fast-export.sh
+++ b/t/t9350-fast-export.sh
@@ -94,14 +94,14 @@ test_expect_success 'fast-export --show-original-ids | git fast-import' '
        test $MUSS = $(git rev-parse --verify refs/tags/muss)
 '
 
-test_expect_success 'iso-8859-7' '
+test_expect_success 'reencoding iso-8859-7' '
 
        test_when_finished "git reset --hard HEAD~1" &&
        test_config i18n.commitencoding iso-8859-7 &&
        test_tick &&
        echo rosten >file &&
        git commit -s -F "$TEST_DIRECTORY/t9350/simple-iso-8859-7-commit-message.txt" file &&
-       git fast-export wer^..wer >iso-8859-7.fi &&
+       git fast-export --reencode=yes wer^..wer >iso-8859-7.fi &&
        sed "s/wer/i18n/" iso-8859-7.fi |
                (cd new &&
                 git fast-import &&
@@ -118,13 +118,45 @@ test_expect_success 'iso-8859-7' '
                 ! grep ^encoding actual)
 '
 
+test_expect_success 'aborting on iso-8859-7' '
+
+       test_when_finished "git reset --hard HEAD~1" &&
+       test_config i18n.commitencoding iso-8859-7 &&
+       echo rosten >file &&
+       git commit -s -F "$TEST_DIRECTORY/t9350/simple-iso-8859-7-commit-message.txt" file &&
+       test_must_fail git fast-export --reencode=abort wer^..wer >iso-8859-7.fi
+'
+
+test_expect_success 'preserving iso-8859-7' '
+
+       test_when_finished "git reset --hard HEAD~1" &&
+       test_config i18n.commitencoding iso-8859-7 &&
+       echo rosten >file &&
+       git commit -s -F "$TEST_DIRECTORY/t9350/simple-iso-8859-7-commit-message.txt" file &&
+       git fast-export --reencode=no wer^..wer >iso-8859-7.fi &&
+       sed "s/wer/i18n-no-recoding/" iso-8859-7.fi |
+               (cd new &&
+                git fast-import &&
+                # The commit object, if not re-encoded, is 240 bytes.
+                # Removing the "encoding iso-8859-7\n" header would drops 20
+                # bytes.  Re-encoding the Pi character from \xF0 (\360) in
+                # iso-8859-7 to \xCF\x80 (\317\200) in UTF-8 adds a byte.
+                # Check for the expected size...
+                test 240 -eq "$(git cat-file -s i18n-no-recoding)" &&
+                # ...as well as the expected byte.
+                git cat-file commit i18n-no-recoding >actual &&
+                grep $(printf "\360") actual &&
+                # Also make sure the commit has the "encoding" header
+                grep ^encoding actual)
+'
+
 test_expect_success 'encoding preserved if reencoding fails' '
 
        test_when_finished "git reset --hard HEAD~1" &&
        test_config i18n.commitencoding iso-8859-7 &&
        echo rosten >file &&
        git commit -s -F "$TEST_DIRECTORY/t9350/broken-iso-8859-7-commit-message.txt" file &&
-       git fast-export wer^..wer >iso-8859-7.fi &&
+       git fast-export --reencode=yes wer^..wer >iso-8859-7.fi &&
        sed "s/wer/i18n-invalid/" iso-8859-7.fi |
                (cd new &&
                 git fast-import &&