QSV: fix some hangs caused by running out of resources.

Be more convervative than before with the full QSV path, and less conservative with encode-only, as it seems to work fine.

git-svn-id: svn://svn.handbrake.fr/HandBrake/branches/qsv@5717 b64f7644-9d1e-0410-96f1-a4d463321fa5

diff --git a/libhb/enc_qsv.c b/libhb/enc_qsv.c
index 284f8f3a88e42ff6b929672ae9070360643af478..c6c609d87176c521ac0fe0a001d588ac56c62ef9 100644 (file)
--- a/libhb/enc_qsv.c
+++ b/libhb/enc_qsv.c
@@ -714,6 +714,23 @@ int encqsvInit(hb_work_object_t *w, hb_job_t *job)
     }
     pv->param.videoParam->mfx.GopPicSize = pv->param.gop.gop_pic_size;
 
+    /* sanitize some settings that affect memory consumption */
+    if (hb_qsv_decode_is_enabled(job))
+    {
+        // limit these to avoid running out of resources (causes hang)
+        pv->param.videoParam->mfx.GopRefDist   = FFMIN(pv->param.videoParam->mfx.GopRefDist,
+                                                       pv->param.rc.lookahead ? 8 : 16);
+        pv->param.codingOption2.LookAheadDepth = FFMIN(pv->param.codingOption2.LookAheadDepth,
+                                                       pv->param.rc.lookahead ? 48 - pv->param.videoParam->mfx.GopRefDist : 0);
+    }
+    else
+    {
+        // encode-only is a bit less sensitive to memory issues
+        pv->param.videoParam->mfx.GopRefDist   = FFMIN(pv->param.videoParam->mfx.GopRefDist, 16);
+        pv->param.codingOption2.LookAheadDepth = FFMIN(pv->param.codingOption2.LookAheadDepth,
+                                                       pv->param.rc.lookahead ? 60 : 0);
+    }
+
     /*
      * init a dummy encode-only session to get the SPS/PPS
      * and the final output settings sanitized by Media SDK

diff --git a/libhb/qsv_common.c b/libhb/qsv_common.c
index 90fa0641a55ba17f42e4c3bdf9a2ca78a3f19fb0..58d05a396dcb87a75c129fe850da97e0cfa7553f 100644 (file)
--- a/libhb/qsv_common.c
+++ b/libhb/qsv_common.c
@@ -703,9 +703,7 @@ int hb_qsv_param_parse(hb_qsv_param_t *param,
         {
             if (!error)
             {
-                // this requires a LOT of video memory - documentation says
-                // 10-100 but limit it to 50 to avoid running out of memory
-                param->codingOption2.LookAheadDepth = HB_QSV_CLIP3(10, 50,
+                param->codingOption2.LookAheadDepth = HB_QSV_CLIP3(10, 100,
                                                                    ivalue);
             }
         }