drm/ast: Validate display modes against framebuffer and format limits

Reimplement ast_mode_config_mode_valid() with DRM format helpers and
ast's helpers for framebuffer size calculation. Replaces ast's open-
coded assumptions on bpp and page-alignments.

Signed-off-by: Thomas Zimmermann <tzimmermann@suse.de>
Reviewed-by: Jocelyn Falempe <jfalempe@redhat.com>
Link: https://lore.kernel.org/r/20250324094520.192974-4-tzimmermann@suse.de

diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index ae6c45f7ff141d2b3f26d5fed3e3342c505adae3..1de832964e9239ea30dd0dc4f1833400be610948 100644 (file)
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -51,6 +51,8 @@
 
 #define AST_LUT_SIZE 256
 
+#define AST_PRIMARY_PLANE_MAX_OFFSET   (BIT(16) - 1)
+
 static unsigned long ast_fb_vram_offset(void)
 {
        return 0; // with shmem, the primary plane is always at offset 0
@@ -960,16 +962,20 @@ static const struct drm_mode_config_helper_funcs ast_mode_config_helper_funcs =
 static enum drm_mode_status ast_mode_config_mode_valid(struct drm_device *dev,
                                                       const struct drm_display_mode *mode)
 {
-       static const unsigned long max_bpp = 4; /* DRM_FORMAT_XRGB8888 */
+       const struct drm_format_info *info = drm_format_info(DRM_FORMAT_XRGB8888);
        struct ast_device *ast = to_ast_device(dev);
-       unsigned long fbsize, fbpages, max_fbpages;
-
-       max_fbpages = ast_fb_vram_size(ast) >> PAGE_SHIFT;
-
-       fbsize = mode->hdisplay * mode->vdisplay * max_bpp;
-       fbpages = DIV_ROUND_UP(fbsize, PAGE_SIZE);
-
-       if (fbpages > max_fbpages)
+       unsigned long max_fb_size = ast_fb_vram_size(ast);
+       u64 pitch;
+
+       if (drm_WARN_ON_ONCE(dev, !info))
+               return MODE_ERROR; /* driver bug */
+
+       pitch = drm_format_info_min_pitch(info, 0, mode->hdisplay);
+       if (!pitch)
+               return MODE_BAD_WIDTH;
+       if (pitch > AST_PRIMARY_PLANE_MAX_OFFSET)
+               return MODE_BAD_WIDTH; /* maximum programmable pitch */
+       if (pitch > max_fb_size / mode->vdisplay)
                return MODE_MEM;
 
        return MODE_OK;