From: Sasha Levin Date: Sun, 16 Feb 2025 15:11:50 +0000 (-0500) Subject: mailbot: cleanup and update X-Git-Tag: v6.12.15~11 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=272b00dccd048f228dbe1dcc7a84d4c64f17cb5c;p=thirdparty%2Fkernel%2Fstable-queue.git mailbot: cleanup and update Signed-off-by: Sasha Levin --- diff --git a/scripts/mailbot.sh b/scripts/mailbot.sh index 8307ec7a0d..cf57764794 100755 --- a/scripts/mailbot.sh +++ b/scripts/mailbot.sh @@ -48,6 +48,76 @@ check_response_exists() { return 1 } +# Function to decode UTF-8 MIME encoded text using Python +decode_mime_header() { + local encoded_text="$1" + [ -z "$encoded_text" ] && return 1 + python3 -c ' +import sys +import email.header +import email.quoprimime +import email.base64mime + +def decode_header(text): + # Handle quoted-printable and base64 encoded UTF-8 headers + while "=?UTF-8?" in text: + start = text.find("=?UTF-8?") + end = text.find("?=", start) + 2 + if end <= 1: # No closing "?=" found + break + + encoded_part = text[start:end] + try: + # Extract encoding type (B or Q) and encoded text + parts = encoded_part.split("?") + if len(parts) != 5: + continue + + charset, encoding, encoded_text = parts[1:4] + if encoding.upper() == "B": + decoded = email.base64mime.decode(encoded_text.encode()) + elif encoding.upper() == "Q": + decoded = email.quoprimime.header_decode(encoded_text) + else: + continue + + # Replace encoded part with decoded text + text = text[:start] + decoded.decode(charset) + text[end:] + except Exception: + # If decoding fails, leave as-is + continue + + return text + +print(decode_header(sys.argv[1])) +' "$encoded_text" 2>/dev/null || echo "$encoded_text" +} + +# Function to normalize author string for comparison +normalize_author() { + local author="$1" + local normalized="" + + # First decode any UTF-8 MIME encoding + normalized=$(decode_mime_header "$author") + + # Remove extra quotes and normalize whitespace + normalized=$(echo "$normalized" | sed -E ' + # Remove surrounding quotes if they exist + s/^"([^"]+)"$/\1/ + # Remove extra spaces around < and > + s/[[:space:]]*<[[:space:]]*/[[:space:]]*/>/g + # Normalize spaces between parts + s/[[:space:]]+/ /g + # Trim leading/trailing whitespace + s/^[[:space:]]+// + s/[[:space:]]+$// + ') + + echo "$normalized" +} + # Function to check if we should ignore this mail based on sender should_ignore_mail() { local mbox_file="$1" @@ -102,19 +172,7 @@ is_git_patch() { # Function to decode UTF-8 email subject decode_subject() { local encoded_subject="$1" - # Check if subject is UTF-8 encoded - if echo "$encoded_subject" | grep -q "=?UTF-8?"; then - # Use perl to decode the subject - echo "$encoded_subject" | perl -MEncode -CS -ne ' - while (/=\?UTF-8\?[Bb]\?([^?]+)\?=/g) { - my $decoded = decode("MIME-Header", $&); - s/\Q$&\E/$decoded/; - } - print; - ' - else - echo "$encoded_subject" - fi + echo "$encoded_subject" | decode_mime_header } # Function to extract series info from subject @@ -122,7 +180,7 @@ extract_series_info() { local subject="$1" # Pattern to match [PATCH X/N] format local part_pattern='\[PATCH.*[[:space:]]([0-9]+)/([0-9]+)\]' - + if [[ $subject =~ $part_pattern ]]; then local current="${BASH_REMATCH[1]}" local total="${BASH_REMATCH[2]}" @@ -281,16 +339,13 @@ extract_kernel_versions() { local subject="$1" local active_versions_file="$HOME/stable-queue/active_kernel_versions" local found_versions=() - local decoded_subject - - decoded_subject=$(decode_subject "$subject") while IFS= read -r version; do - # Check for both with and without v prefix - if echo "$decoded_subject" | grep -q "\\b${version}\\b\|${version}[.-]\|\\bv${version}\\b\|v${version}[.-]"; then + # Check for version numbers with optional v prefix in subject + if printf "%s" "$subject" | grep -E -q "(^|[^0-9])${version}([^0-9]|$)|(^|[[:space:]]|\[)v${version}([^0-9]|$)"; then found_versions+=("$version") fi - done < <(sort -rV "$active_versions_file") + done < "$active_versions_file" if [ ${#found_versions[@]} -gt 0 ]; then echo "${found_versions[*]}" @@ -324,13 +379,43 @@ extract_commit_sha1() { return 1 } -# Function to get commit author information +# Function to extract patch author with normalization +extract_patch_author() { + local mbox_file="$1" + local author=$(formail -xFrom: < "$mbox_file") + normalize_author "$author" +} + +# Function to get commit author with normalization get_commit_author() { local linux_dir="$1" local sha1="$2" cd "$linux_dir" - git log -1 --format="%an <%ae>" "$sha1" + local author=$(git log -1 --format="%an <%ae>" "$sha1") + normalize_author "$author" +} + +# Function to compare authors and check if they match +authors_match() { + local author1="$1" + local author2="$2" + + # Normalize both authors + local norm1=$(normalize_author "$author1") + local norm2=$(normalize_author "$author2") + + # Extract email parts for comparison + local email1=$(echo "$norm1" | grep -o '<[^>]*>' || echo "") + local email2=$(echo "$norm2" | grep -o '<[^>]*>' || echo "") + + # If emails match and are non-empty, authors match + if [ -n "$email1" ] && [ "$email1" = "$email2" ]; then + return 0 + fi + + # If emails don't match or are empty, compare full normalized strings + [ "$norm1" = "$norm2" ] } # Function to get sorted kernel versions @@ -342,7 +427,7 @@ get_sorted_versions() { is_version_newer() { local v1="$1" local v2="$2" - + if [ "$(echo -e "$v1\n$v2" | sort -V | tail -n1)" = "$v1" ]; then return 0 fi @@ -355,32 +440,33 @@ check_newer_kernels() { local target_versions="$2" local linux_dir="$3" local -n c_results=$4 - + cd "$linux_dir" local all_versions=($(get_sorted_versions)) local target_array=($target_versions) local newest_target=${target_array[0]} local temp_dir=$(mktemp -d) local pids=() - + local checked_versions=() + # Function to check a single branch and write results to temp file check_single_branch() { local version="$1" local branch="pending-${version}" local result_file="$temp_dir/$version" - + # Check if branch exists if ! git rev-parse --verify "$branch" >/dev/null 2>&1; then echo "$version.y | Branch not found" > "$result_file" return fi - + # Check if commit is an ancestor using merge-base if [ "$(git merge-base "$branch" "$sha1")" = "$sha1" ]; then echo "$version.y | Present (exact SHA1)" > "$result_file" return fi - + # Try to find by subject if SHA1 not found local subject subject=$(git log -1 --format=%s "$sha1") @@ -397,38 +483,39 @@ check_newer_kernels() { echo "$version.y | Not found" > "$result_file" fi } - + # Launch parallel processes for each relevant version for version in "${all_versions[@]}"; do - if is_version_newer "$version" "$newest_target"; then + # Only check versions newer than our target using bc for proper numeric comparison + if (( $(echo "$version > $newest_target" | bc -l) )); then check_single_branch "$version" & pids+=($!) + checked_versions+=("$version") fi done - + # Wait for all processes to complete for pid in "${pids[@]}"; do wait "$pid" done - + # Collect results in order local results=() - for version in "${all_versions[@]}"; do - if is_version_newer "$version" "$newest_target"; then - if [ -f "$temp_dir/$version" ]; then - results+=("$(cat "$temp_dir/$version")") - fi + for version in "${checked_versions[@]}"; do + if [ -f "$temp_dir/$version" ]; then + results+=("$(cat "$temp_dir/$version")") fi done - - # Clean up - rm -rf "$temp_dir" - + + # Add results only if we found newer versions if [ ${#results[@]} -gt 0 ]; then c_results+=("") c_results+=("Status in newer kernel trees:") c_results+=("${results[@]}") fi + + # Clean up + rm -rf "$temp_dir" } # Function to validate commit exists in upstream @@ -452,44 +539,44 @@ compare_with_upstream() { fi cd "$linux_dir" - + # Extract subject to determine target kernel versions local subject=$(formail -xSubject: < "$mbox_file") local kernel_versions=($(extract_kernel_versions "$subject")) - + if [ ${#kernel_versions[@]} -eq 0 ]; then debug_output+=("No kernel versions found in subject, cannot compare") printf '%s\n' "${debug_output[@]}" return 1 fi - + # Use the newest version for comparison since it's closest to upstream local version=${kernel_versions[0]} local stable_branch="stable/linux-${version}.y" local temp_branch="temp-compare-${version}-$(date +%s)" local current_branch=$(git rev-parse --abbrev-ref HEAD) - + # Try to check out the stable branch if ! git checkout -q "$stable_branch" 2>/dev/null; then debug_output+=("Failed to find stable branch ${stable_branch}") printf '%s\n' "${debug_output[@]}" return 1 fi - + # Create temporary branch based on the stable branch git checkout -b "$temp_branch" >/dev/null 2>&1 - + if git am "$mbox_file" >/dev/null 2>&1; then # Get the SHA1 of our newly applied patch local new_sha1=$(git rev-parse HEAD) - + debug_output+=("") - + # Compare the ranges using range-diff if ! git range-diff "${sha1}^".."$sha1" "${new_sha1}^".."$new_sha1"; then debug_output+=("Failed to generate range-diff") fi - + # Clean up after range-diff git checkout -q "$current_branch" git branch -D "$temp_branch" >/dev/null 2>&1 @@ -498,29 +585,29 @@ compare_with_upstream() { git am --abort >/dev/null 2>&1 git checkout -q "$current_branch" git branch -D "$temp_branch" >/dev/null 2>&1 - + debug_output+=("Failed to apply patch cleanly, falling back to interdiff...") debug_output+=("") - + # Fall back to interdiff TEMP_PATCH=$(mktemp) UPSTREAM_PATCH=$(mktemp) FILTERED_PATCH=$(mktemp) FILTERED_UPSTREAM=$(mktemp) - + # Extract patch content from mbox formail -I "" < "$mbox_file" | sed '1,/^$/d' > "$TEMP_PATCH" - + # Get upstream patch git format-patch -k --stdout --no-signature "${sha1}^..${sha1}" | \ sed '1,/^$/d' > "$UPSTREAM_PATCH" - + # Filter out metadata lines that might cause comparison issues for file in "$TEMP_PATCH" "$UPSTREAM_PATCH"; do output_file="${file/PATCH/FILTERED}" sed -E '/^(index |diff --git |new file mode |deleted file mode |old mode |new mode )/d' "$file" > "$output_file" done - + # Run interdiff if ! interdiff "$FILTERED_UPSTREAM" "$FILTERED_PATCH" 2>"${TEMP_PATCH}.err"; then # If interdiff fails, include error output @@ -528,7 +615,7 @@ compare_with_upstream() { debug_output+=("interdiff error output:") debug_output+=("$(cat "${TEMP_PATCH}.err")") fi - + # Fall back to standard diff debug_output+=("interdiff failed, falling back to standard diff...") if ! diff -u "$FILTERED_UPSTREAM" "$FILTERED_PATCH" >"${TEMP_PATCH}.diff"; then @@ -537,11 +624,11 @@ compare_with_upstream() { fi fi fi - + # Clean up temporary files rm -f "$TEMP_PATCH" "$UPSTREAM_PATCH" "$FILTERED_PATCH" "$FILTERED_UPSTREAM" "${TEMP_PATCH}.err" "${TEMP_PATCH}.diff" fi - + printf '%s\n' "${debug_output[@]}" } @@ -589,7 +676,7 @@ test_commit_on_branch() { fi # Run build test - if ! ~/pulls/build-next.sh; then + if ! stable build; then if [ -f ~/errors-linus-next ]; then local build_error=$(cat ~/errors-linus-next) results+=("stable/linux-${version}.y | Success | Failed") @@ -610,10 +697,58 @@ test_commit_on_branch() { return $result } -# Function to extract patch author -extract_patch_author() { - local mbox_file="$1" - formail -xFrom: < "$mbox_file" | sed -e 's/^[ \t]*//' +# Function to check for fixes referencing a commit +check_fixes_for_commit() { + local sha1="$1" + local linux_dir="$2" + local -n result_array=$3 + + cd "$linux_dir" + + # Look for commits with Fixes: tag pointing to our commit + local fixes_commits=$(git log origin/master --grep="Fixes: ${sha1:0:12}" --format="%H %s") + + if [ -n "$fixes_commits" ]; then + result_array+=("Found fixes commits:") + while IFS= read -r line; do + local fix_sha1="${line%% *}" + local fix_subject="${line#* }" + result_array+=("${fix_sha1:0:12} ${fix_subject}") + done <<< "$fixes_commits" + fi +} + +# Function to check if commit was reverted +check_if_reverted() { + local sha1="$1" + local linux_dir="$2" + local -n result_array=$3 + + cd "$linux_dir" + + # Look for revert commits in subject and body + local revert_commits=$(git log origin/master --grep="This reverts commit ${sha1:0:12}\|^Revert \".*${sha1:0:12}.*\"" --format="%H %s") + + # Also look for Fixes: tags in revert commits + local fixes_reverts=$(git log origin/master --grep="^Revert.*\|Fixes: ${sha1:0:12}" --format="%H %B" | \ + awk -v sha="$sha1" ' + /^[0-9a-f]{40}/ { commit=$1; subject=$0; sub(/^[0-9a-f]{40}[[:space:]]*/, "", subject); } + /^Revert/ && /Fixes: '"${sha1:0:12}"'/ { print commit " " subject; } + ') + + # Combine both results + local all_reverts=$(printf "%s\n%s" "$revert_commits" "$fixes_reverts" | sort -u) + + if [ -n "$all_reverts" ]; then + result_array+=("Found revert commits:") + while IFS= read -r line; do + if [ -n "$line" ]; then + local revert_sha1="${line%% *}" + local revert_subject="${line#* }" + result_array+=("${revert_sha1:0:12} ${revert_subject}") + fi + done <<< "$all_reverts" + fi } # Function to process single patch or series patch @@ -631,9 +766,9 @@ process_patch() { cd "$LINUX_DIR" local temp_branch="temp-series-check-$(date +%s)" local current_branch=$(git rev-parse --abbrev-ref HEAD) - + git checkout -b "$temp_branch" >/dev/null 2>&1 - + if ! apply_series_patches "$series_dir" "$current_part" "$LINUX_DIR"; then git checkout -q "$current_branch" git branch -D "$temp_branch" >/dev/null 2>&1 @@ -641,7 +776,7 @@ process_patch() { p_errors+=("Error: Cannot proceed - previous patches in series failed to apply") return 1 fi - + git checkout -q "$current_branch" git branch -D "$temp_branch" >/dev/null 2>&1 fi @@ -677,8 +812,9 @@ process_patch() { if [ -n "$found_sha1" ] && [ "$found_sha1" != "0000000000000000000000000000000000000000" ]; then local patch_author=$(extract_patch_author "$mbox_file") local commit_author=$(get_commit_author "$LINUX_DIR" "$found_sha1") - if [ "$patch_author" != "$commit_author" ]; then - author_mismatch="Backport author: $patch_author"$'\n'"Commit author: $commit_author" + if ! authors_match "$patch_author" "$commit_author"; then + author_mismatch="Backport author: $patch_author +Commit author: $commit_author" fi fi @@ -701,11 +837,21 @@ process_patch() { check_newer_kernels "$found_sha1" "$kernel_versions" "$LINUX_DIR" newer_kernel_results fi + # Check for fixes and reverts if we have a valid SHA1 + local -a fixes_results=() + local -a revert_results=() + if [[ "$found_sha1" =~ ^[0-9a-f]{40}$ ]] && [ "$found_sha1" != "0000000000000000000000000000000000000000" ]; then + check_fixes_for_commit "$found_sha1" "$LINUX_DIR" fixes_results + check_if_reverted "$found_sha1" "$LINUX_DIR" revert_results + fi + # Generate response for this patch generate_response "$mbox_file" "$claimed_sha1" "$found_sha1" \ "$(printf '%s\n' "${p_results[@]}")" "$diff_output" \ "$author_mismatch" "$(printf '%s\n' "${p_errors[@]}")" \ - "$(printf '%s\n' "${newer_kernel_results[@]}")" + "$(printf '%s\n' "${newer_kernel_results[@]}")" \ + "$(printf '%s\n' "${fixes_results[@]}")" \ + "$(printf '%s\n' "${revert_results[@]}")" return $failed } @@ -743,13 +889,15 @@ generate_response() { local author_mismatch="$6" local build_errors="$7" local newer_kernel_results="$8" + local fixes_results="$9" + local revert_results="${10}" local response_file=$(generate_response_filename "$mbox_file") { # Get the From, Subject, Message-ID, and Date from original email for threading formail -X From: -X Subject: < "$mbox_file" - echo "Message-ID: $(generate_message_id)" - echo "Date: $(date -R)" + echo "Message-ID: $(generate_message_id)" + echo "Date: $(date -R)" echo "In-Reply-To: $(formail -xMessage-ID: < "$mbox_file")" echo "From: $(git config user.name) <$(git config user.email)>" @@ -766,12 +914,108 @@ generate_response() { echo "Subject: $orig_subject" fi + # Add summary section to determine if there are issues + local has_issues=0 + local summary=() + + # Check for build failures + if [[ "$results" == *"| Failed"* ]]; then + has_issues=1 + summary+=("❌ Build failures detected") + fi + + # Check for missing or unverified commit + if [ -z "$found_sha1" ] || [ "$found_sha1" = "0000000000000000000000000000000000000000" ]; then + has_issues=1 + summary+=("⚠️ Could not find matching upstream commit") + elif [ -n "$claimed_sha1" ] && [ "$claimed_sha1" != "$found_sha1" ]; then + has_issues=1 + summary+=("⚠️ Provided upstream commit SHA1 does not match found commit") + elif [ -z "$claimed_sha1" ] && [ -n "$found_sha1" ]; then + has_issues=1 + summary+=("⚠️ Found matching upstream commit but patch is missing proper reference to it") + fi + + # Check for fixes + if [ -n "$fixes_results" ]; then + has_issues=1 + summary+=("⚠️ Found follow-up fixes in mainline") + fi + + # Check for reverts + if [ -n "$revert_results" ]; then + has_issues=1 + summary+=("❌ Commit was reverted in mainline") + fi + + # Check for missing commits in newer stable branches + if [ -n "$newer_kernel_results" ]; then + local missing_in_newer=0 + local missing_count=0 + local newer_count=0 + + # Extract target version from results + local target_version="" + while IFS='|' read -r branch status build; do + if [[ "$branch" =~ linux-([0-9]+\.[0-9]+)\.y ]]; then + target_version="${BASH_REMATCH[1]}" + break + fi + done <<< "$results" + + # Only check newer branches + while IFS= read -r line; do + if [[ "$line" =~ ^([0-9]+\.[0-9]+)\.y ]]; then + local branch_version="${BASH_REMATCH[1]}" + # Compare versions using bc for proper numeric comparison + if (( $(echo "$branch_version > $target_version" | bc -l) )); then + newer_count=$((newer_count + 1)) + if [[ "$line" == *"| Not found"* ]]; then + missing_count=$((missing_count + 1)) + fi + fi + fi + done <<< "$newer_kernel_results" + + # Only set missing_in_newer if ALL newer branches are missing the commit + if [ $missing_count -gt 0 ] && [ $missing_count -eq $newer_count ]; then + missing_in_newer=1 + has_issues=1 + summary+=("⚠️ Commit missing in some newer stable branches") + fi + fi + + # Add appropriate email headers based on whether there are issues + if [ $has_issues -eq 1 ]; then + # If there are issues, send to author and CC stable + local author_email=$(formail -xFrom: < "$mbox_file" | sed -e 's/.*<\([^>]*\)>.*/\1/') + echo "To: $author_email" + echo "Cc: stable@vger.kernel.org" + else + # If no issues, send only to stable + echo "To: stable@vger.kernel.org" + fi echo + echo "[ Sasha's backport helper bot ]" echo echo "Hi," echo + # Print summary section + if [ $has_issues -eq 1 ]; then + echo "Summary of potential issues:" + printf '%s\n' "${summary[@]}" + echo + else + # Check if patch applies and builds successfully + if [[ "$results" == *"| Success | Success"* ]]; then + echo "✅ All tests passed successfully. No issues detected." + echo "No action required from the submitter." + echo + fi + fi + # Report on SHA1 verification and commit status if [ -n "$claimed_sha1" ]; then if [ "$claimed_sha1" = "$found_sha1" ]; then @@ -810,6 +1054,17 @@ generate_response() { echo fi + # Add fixes and revert information if available + if [ -n "$fixes_results" ]; then + echo "$fixes_results" + echo + fi + + if [ -n "$revert_results" ]; then + echo "$revert_results" + echo + fi + # Add diff if there are differences and we have a valid SHA1 if [ -n "$diff_output" ] && [[ "$found_sha1" =~ ^[0-9a-f]{40}$ ]] && \ [ "$found_sha1" != "0000000000000000000000000000000000000000" ]; then