st/mesa: fix 2 crashes in st_tgsi_lower_yuv

Submitted by Marek Olšák on May 10, 2019, 8:28 p.m.

Details

Message ID 20190510202856.26944-1-maraeo@gmail.com
State New
Headers show
Series "st/mesa: fix 2 crashes in st_tgsi_lower_yuv" ( rev: 1 ) in Mesa

Not browsing as part of any series.

Commit Message

Marek Olšák May 10, 2019, 8:28 p.m.
From: Marek Olšák <marek.olsak@amd.com>

src/mesa/state_tracker/st_tgsi_lower_yuv.c:68: void reg_dst(struct
 tgsi_full_dst_register *, const struct tgsi_full_dst_register *, unsigned
 int): assertion "dst->Register.WriteMask" failed

The second crash was due to insufficient allocated size for TGSI
instructions.

Cc: 19.0 19.1 <mesa-stable@lists.freedesktop.org>
---
 src/mesa/state_tracker/st_tgsi_lower_yuv.c | 48 +++++++++++++---------
 1 file changed, 28 insertions(+), 20 deletions(-)

Patch hide | download patch | download mbox

diff --git a/src/mesa/state_tracker/st_tgsi_lower_yuv.c b/src/mesa/state_tracker/st_tgsi_lower_yuv.c
index 6acd173adc9..73437ddda70 100644
--- a/src/mesa/state_tracker/st_tgsi_lower_yuv.c
+++ b/src/mesa/state_tracker/st_tgsi_lower_yuv.c
@@ -262,45 +262,53 @@  yuv_to_rgb(struct tgsi_transform_context *tctx,
    inst.Instruction.Saturate = 0;
    inst.Instruction.NumDstRegs = 1;
    inst.Instruction.NumSrcRegs = 2;
    reg_dst(&inst.Dst[0], &ctx->tmp[A].dst, TGSI_WRITEMASK_XYZ);
    reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, _));
    reg_src(&inst.Src[1], &ctx->imm[3], SWIZ(X, Y, Z, _));
    inst.Src[1].Register.Negate = 1;
    tctx->emit_instruction(tctx, &inst);
 
    /* DP3 dst.x, tmpA, imm[0] */
-   inst = dp3_instruction();
-   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_X);
-   reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
-   reg_src(&inst.Src[1], &ctx->imm[0], SWIZ(X, Y, Z, W));
-   tctx->emit_instruction(tctx, &inst);
+   if (dst->Register.WriteMask & TGSI_WRITEMASK_X) {
+      inst = dp3_instruction();
+      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_X);
+      reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
+      reg_src(&inst.Src[1], &ctx->imm[0], SWIZ(X, Y, Z, W));
+      tctx->emit_instruction(tctx, &inst);
+   }
 
    /* DP3 dst.y, tmpA, imm[1] */
-   inst = dp3_instruction();
-   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Y);
-   reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
-   reg_src(&inst.Src[1], &ctx->imm[1], SWIZ(X, Y, Z, W));
-   tctx->emit_instruction(tctx, &inst);
+   if (dst->Register.WriteMask & TGSI_WRITEMASK_Y) {
+      inst = dp3_instruction();
+      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Y);
+      reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
+      reg_src(&inst.Src[1], &ctx->imm[1], SWIZ(X, Y, Z, W));
+      tctx->emit_instruction(tctx, &inst);
+   }
 
    /* DP3 dst.z, tmpA, imm[2] */
-   inst = dp3_instruction();
-   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Z);
-   reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
-   reg_src(&inst.Src[1], &ctx->imm[2], SWIZ(X, Y, Z, W));
-   tctx->emit_instruction(tctx, &inst);
+   if (dst->Register.WriteMask & TGSI_WRITEMASK_Z) {
+      inst = dp3_instruction();
+      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Z);
+      reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
+      reg_src(&inst.Src[1], &ctx->imm[2], SWIZ(X, Y, Z, W));
+      tctx->emit_instruction(tctx, &inst);
+   }
 
    /* MOV dst.w, imm[0].x */
-   inst = mov_instruction();
-   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_W);
-   reg_src(&inst.Src[0], &ctx->imm[3], SWIZ(_, _, _, W));
-   tctx->emit_instruction(tctx, &inst);
+   if (dst->Register.WriteMask & TGSI_WRITEMASK_W) {
+      inst = mov_instruction();
+      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_W);
+      reg_src(&inst.Src[0], &ctx->imm[3], SWIZ(_, _, _, W));
+      tctx->emit_instruction(tctx, &inst);
+   }
 }
 
 static void
 lower_nv12(struct tgsi_transform_context *tctx,
            struct tgsi_full_instruction *originst)
 {
    struct tgsi_yuv_transform *ctx = tgsi_yuv_transform(tctx);
    struct tgsi_full_instruction inst;
    struct tgsi_full_src_register *coord = &originst->Src[0];
    unsigned samp = originst->Src[1].Register.Index;
@@ -427,21 +435,21 @@  st_tgsi_lower_yuv(const struct tgsi_token *tokens, unsigned free_slots,
    memset(&ctx, 0, sizeof(ctx));
    ctx.base.transform_instruction = transform_instr;
    ctx.free_slots = free_slots;
    ctx.lower_nv12 = lower_nv12;
    ctx.lower_iyuv = lower_iyuv;
    tgsi_scan_shader(tokens, &ctx.info);
 
    /* TODO better job of figuring out how many extra tokens we need..
     * this is a pain about tgsi_transform :-/
     */
-   newlen = tgsi_num_tokens(tokens) + 120;
+   newlen = tgsi_num_tokens(tokens) + 300;
    newtoks = tgsi_alloc_tokens(newlen);
    if (!newtoks)
       return NULL;
 
    tgsi_transform_shader(tokens, newtoks, newlen, &ctx.base);
 
 //   tgsi_dump(newtoks, 0);
 //   debug_printf("\n");
 
    return newtoks;

Comments

On Fri, May 10, 2019 at 1:29 PM Marek Olšák <maraeo@gmail.com> wrote:
>
> From: Marek Olšák <marek.olsak@amd.com>
>
> src/mesa/state_tracker/st_tgsi_lower_yuv.c:68: void reg_dst(struct
>  tgsi_full_dst_register *, const struct tgsi_full_dst_register *, unsigned
>  int): assertion "dst->Register.WriteMask" failed
>
> The second crash was due to insufficient allocated size for TGSI
> instructions.

I might have split this into two patches, but either way you can add my:

Reviewed-by: Rob Clark <robdclark@gmail.com>

>
> Cc: 19.0 19.1 <mesa-stable@lists.freedesktop.org>
> ---
>  src/mesa/state_tracker/st_tgsi_lower_yuv.c | 48 +++++++++++++---------
>  1 file changed, 28 insertions(+), 20 deletions(-)
>
> diff --git a/src/mesa/state_tracker/st_tgsi_lower_yuv.c b/src/mesa/state_tracker/st_tgsi_lower_yuv.c
> index 6acd173adc9..73437ddda70 100644
> --- a/src/mesa/state_tracker/st_tgsi_lower_yuv.c
> +++ b/src/mesa/state_tracker/st_tgsi_lower_yuv.c
> @@ -262,45 +262,53 @@ yuv_to_rgb(struct tgsi_transform_context *tctx,
>     inst.Instruction.Saturate = 0;
>     inst.Instruction.NumDstRegs = 1;
>     inst.Instruction.NumSrcRegs = 2;
>     reg_dst(&inst.Dst[0], &ctx->tmp[A].dst, TGSI_WRITEMASK_XYZ);
>     reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, _));
>     reg_src(&inst.Src[1], &ctx->imm[3], SWIZ(X, Y, Z, _));
>     inst.Src[1].Register.Negate = 1;
>     tctx->emit_instruction(tctx, &inst);
>
>     /* DP3 dst.x, tmpA, imm[0] */
> -   inst = dp3_instruction();
> -   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_X);
> -   reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
> -   reg_src(&inst.Src[1], &ctx->imm[0], SWIZ(X, Y, Z, W));
> -   tctx->emit_instruction(tctx, &inst);
> +   if (dst->Register.WriteMask & TGSI_WRITEMASK_X) {
> +      inst = dp3_instruction();
> +      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_X);
> +      reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
> +      reg_src(&inst.Src[1], &ctx->imm[0], SWIZ(X, Y, Z, W));
> +      tctx->emit_instruction(tctx, &inst);
> +   }
>
>     /* DP3 dst.y, tmpA, imm[1] */
> -   inst = dp3_instruction();
> -   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Y);
> -   reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
> -   reg_src(&inst.Src[1], &ctx->imm[1], SWIZ(X, Y, Z, W));
> -   tctx->emit_instruction(tctx, &inst);
> +   if (dst->Register.WriteMask & TGSI_WRITEMASK_Y) {
> +      inst = dp3_instruction();
> +      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Y);
> +      reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
> +      reg_src(&inst.Src[1], &ctx->imm[1], SWIZ(X, Y, Z, W));
> +      tctx->emit_instruction(tctx, &inst);
> +   }
>
>     /* DP3 dst.z, tmpA, imm[2] */
> -   inst = dp3_instruction();
> -   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Z);
> -   reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
> -   reg_src(&inst.Src[1], &ctx->imm[2], SWIZ(X, Y, Z, W));
> -   tctx->emit_instruction(tctx, &inst);
> +   if (dst->Register.WriteMask & TGSI_WRITEMASK_Z) {
> +      inst = dp3_instruction();
> +      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_Z);
> +      reg_src(&inst.Src[0], &ctx->tmp[A].src, SWIZ(X, Y, Z, W));
> +      reg_src(&inst.Src[1], &ctx->imm[2], SWIZ(X, Y, Z, W));
> +      tctx->emit_instruction(tctx, &inst);
> +   }
>
>     /* MOV dst.w, imm[0].x */
> -   inst = mov_instruction();
> -   reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_W);
> -   reg_src(&inst.Src[0], &ctx->imm[3], SWIZ(_, _, _, W));
> -   tctx->emit_instruction(tctx, &inst);
> +   if (dst->Register.WriteMask & TGSI_WRITEMASK_W) {
> +      inst = mov_instruction();
> +      reg_dst(&inst.Dst[0], dst, TGSI_WRITEMASK_W);
> +      reg_src(&inst.Src[0], &ctx->imm[3], SWIZ(_, _, _, W));
> +      tctx->emit_instruction(tctx, &inst);
> +   }
>  }
>
>  static void
>  lower_nv12(struct tgsi_transform_context *tctx,
>             struct tgsi_full_instruction *originst)
>  {
>     struct tgsi_yuv_transform *ctx = tgsi_yuv_transform(tctx);
>     struct tgsi_full_instruction inst;
>     struct tgsi_full_src_register *coord = &originst->Src[0];
>     unsigned samp = originst->Src[1].Register.Index;
> @@ -427,21 +435,21 @@ st_tgsi_lower_yuv(const struct tgsi_token *tokens, unsigned free_slots,
>     memset(&ctx, 0, sizeof(ctx));
>     ctx.base.transform_instruction = transform_instr;
>     ctx.free_slots = free_slots;
>     ctx.lower_nv12 = lower_nv12;
>     ctx.lower_iyuv = lower_iyuv;
>     tgsi_scan_shader(tokens, &ctx.info);
>
>     /* TODO better job of figuring out how many extra tokens we need..
>      * this is a pain about tgsi_transform :-/
>      */
> -   newlen = tgsi_num_tokens(tokens) + 120;
> +   newlen = tgsi_num_tokens(tokens) + 300;
>     newtoks = tgsi_alloc_tokens(newlen);
>     if (!newtoks)
>        return NULL;
>
>     tgsi_transform_shader(tokens, newtoks, newlen, &ctx.base);
>
>  //   tgsi_dump(newtoks, 0);
>  //   debug_printf("\n");
>
>     return newtoks;
> --
> 2.17.1
>
> _______________________________________________
> mesa-dev mailing list
> mesa-dev@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/mesa-dev