Merge branch 'peace_love_and_stencils' into 'master'

Support morrowind stenciling (#6443)

Closes #6443

See merge request OpenMW/openmw!1635

CHANGELOG.md

@@ -129,6 +129,7 @@
     Feature #6288: Preserve the "blocked" record flag for referenceable objects.
     Feature #6380: Commas are treated as whitespace in vanilla
     Feature #6419: Topics shouldn't be greyed out if they can produce another topic reference
+    Feature #6443: NiStencilProperty is not fully supported
     Feature #6534: Shader-based object texture blending
     Task #6201: Remove the "Note: No relevant classes found. No output generated" warnings
     Task #6264: Remove the old classes in animation.cpp

apps/openmw/mwrender/localmap.cpp

@@ -187,7 +187,7 @@ osg::ref_ptr<osg::Camera> LocalMap::createOrthographicCamera(float x, float y, f
     camera->setReferenceFrame(osg::Camera::ABSOLUTE_RF_INHERIT_VIEWPOINT);
     camera->setRenderTargetImplementation(osg::Camera::FRAME_BUFFER_OBJECT, osg::Camera::PIXEL_BUFFER_RTT);
     camera->setClearColor(osg::Vec4(0.f, 0.f, 0.f, 1.f));
-    camera->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+    camera->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
     camera->setRenderOrder(osg::Camera::PRE_RENDER);
 
     camera->setCullMask(Mask_Scene | Mask_SimpleWater | Mask_Terrain | Mask_Object | Mask_Static);

apps/openmw/mwrender/npcanimation.cpp

@@ -333,12 +333,12 @@ public:
 
             osg::FrameBufferAttachment(postProcessor->getFirstPersonRBProxy()).attach(*state, GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT_EXT, ext);
 
-            glClear(GL_DEPTH_BUFFER_BIT);
+            glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
             // color accumulation pass
             bin->drawImplementation(renderInfo, previous);
 
             auto primaryFBO = postProcessor->getMsaaFbo() ? postProcessor->getMsaaFbo() : postProcessor->getFbo();
-            primaryFBO->getAttachment(osg::FrameBufferObject::BufferComponent::DEPTH_BUFFER).attach(*state, GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT_EXT, ext);
+            primaryFBO->getAttachment(osg::FrameBufferObject::BufferComponent::PACKED_DEPTH_STENCIL_BUFFER).attach(*state, GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT_EXT, ext);
 
             state->pushStateSet(mStateSet);
             state->apply();
@@ -349,7 +349,7 @@ public:
         else
         {
             // fallback to standard depth clear when we are not rendering our main scene via an intermediate FBO
-            glClear(GL_DEPTH_BUFFER_BIT);
+            glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
             bin->drawImplementation(renderInfo, previous);
         }
     }

apps/openmw/mwrender/postprocessor.cpp

@@ -1,5 +1,7 @@
 #include "postprocessor.hpp"
 
+#include <SDL_opengl_glext.h>
+
 #include <osg/Group>
 #include <osg/Camera>
 #include <osg/Callback>
@@ -155,7 +157,7 @@ namespace MWRender
     PostProcessor::PostProcessor(osgViewer::Viewer* viewer, osg::Group* rootNode)
         : mViewer(viewer)
         , mRootNode(new osg::Group)
-        , mDepthFormat(GL_DEPTH_COMPONENT24)
+        , mDepthFormat(GL_DEPTH24_STENCIL8_EXT)
     {
         bool softParticles = Settings::Manager::getBool("soft particles", "Shaders");
 
@@ -183,9 +185,9 @@ namespace MWRender
         if (SceneUtil::AutoDepth::isReversed())
         {
             if (osg::isGLExtensionSupported(contextID, "GL_ARB_depth_buffer_float"))
-                mDepthFormat = GL_DEPTH_COMPONENT32F;
+                mDepthFormat = GL_DEPTH32F_STENCIL8;
             else if (osg::isGLExtensionSupported(contextID, "GL_NV_depth_buffer_float"))
-                mDepthFormat = GL_DEPTH_COMPONENT32F_NV;
+                mDepthFormat = GL_DEPTH32F_STENCIL8_NV;
             else
             {
                 // TODO: Once we have post-processing implemented we want to skip this return and continue with setup.
@@ -213,7 +215,7 @@ namespace MWRender
         mViewer->getCamera()->addCullCallback(new CullCallback);
         mViewer->getCamera()->setRenderTargetImplementation(osg::Camera::FRAME_BUFFER_OBJECT);
         mViewer->getCamera()->attach(osg::Camera::COLOR_BUFFER0, mSceneTex);
-        mViewer->getCamera()->attach(osg::Camera::DEPTH_BUFFER, mDepthTex);
+        mViewer->getCamera()->attach(osg::Camera::PACKED_DEPTH_STENCIL_BUFFER, mDepthTex);
 
         mViewer->getCamera()->getGraphicsContext()->setResizedCallback(new ResizedCallback(this));
         mViewer->getCamera()->setUserData(this);
@@ -236,7 +238,7 @@ namespace MWRender
 
         mFbo = new osg::FrameBufferObject;
         mFbo->setAttachment(osg::Camera::COLOR_BUFFER0, osg::FrameBufferAttachment(mSceneTex));
-        mFbo->setAttachment(osg::Camera::DEPTH_BUFFER, osg::FrameBufferAttachment(mDepthTex));
+        mFbo->setAttachment(osg::Camera::PACKED_DEPTH_STENCIL_BUFFER, osg::FrameBufferAttachment(mDepthTex));
 
         // When MSAA is enabled we must first render to a render buffer, then
         // blit the result to the FBO which is either passed to the main frame
@@ -247,7 +249,7 @@ namespace MWRender
             osg::ref_ptr<osg::RenderBuffer> colorRB = new osg::RenderBuffer(width, height, mSceneTex->getInternalFormat(), samples);
             osg::ref_ptr<osg::RenderBuffer> depthRB = new osg::RenderBuffer(width, height, mDepthTex->getInternalFormat(), samples);
             mMsaaFbo->setAttachment(osg::Camera::COLOR_BUFFER0, osg::FrameBufferAttachment(colorRB));
-            mMsaaFbo->setAttachment(osg::Camera::DEPTH_BUFFER, osg::FrameBufferAttachment(depthRB));
+            mMsaaFbo->setAttachment(osg::Camera::PACKED_DEPTH_STENCIL_BUFFER, osg::FrameBufferAttachment(depthRB));
         }
 
         if (const auto depthProxy = std::getenv("OPENMW_ENABLE_DEPTH_CLEAR_PROXY"))
@@ -264,8 +266,8 @@ namespace MWRender
     {
         mDepthTex = new osg::Texture2D;
         mDepthTex->setTextureSize(width, height);
-        mDepthTex->setSourceFormat(GL_DEPTH_COMPONENT);
+        mDepthTex->setSourceFormat(GL_DEPTH_STENCIL_EXT);
-        mDepthTex->setSourceType(SceneUtil::isFloatingPointDepthFormat(getDepthFormat()) ? GL_FLOAT : GL_UNSIGNED_INT);
+        mDepthTex->setSourceType(SceneUtil::isFloatingPointDepthFormat(getDepthFormat()) ? GL_FLOAT_32_UNSIGNED_INT_24_8_REV : GL_UNSIGNED_INT_24_8_EXT);
         mDepthTex->setInternalFormat(mDepthFormat);
         mDepthTex->setFilter(osg::Texture2D::MIN_FILTER, osg::Texture2D::NEAREST);
         mDepthTex->setFilter(osg::Texture2D::MAG_FILTER, osg::Texture2D::NEAREST);

apps/openmw/mwrender/renderingmanager.cpp

@@ -538,6 +538,8 @@ namespace MWRender
         SceneUtil::setCameraClearDepth(mViewer->getCamera());
 
         updateProjectionMatrix();
+
+        mViewer->getCamera()->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
     }
 
     RenderingManager::~RenderingManager()

apps/openmw/mwrender/screenshotmanager.cpp

@@ -348,7 +348,7 @@ namespace MWRender
 
         rttCamera->setCullMask(mViewer->getCamera()->getCullMask() & ~(Mask_GUI|Mask_FirstPerson));
 
-        rttCamera->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+        rttCamera->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 
         renderCameraToImage(rttCamera.get(),image,w,h);
     }

components/nifosg/nifloader.cpp

@@ -218,6 +218,7 @@ namespace NifOsg
 
         bool mHasNightDayLabel = false;
         bool mHasHerbalismLabel = false;
+        bool mHasStencilProperty = false;
 
         // This is used to queue emitters that weren't attached to their node yet.
         std::vector<std::pair<size_t, osg::ref_ptr<Emitter>>> mEmitterQueue;
@@ -309,6 +310,10 @@ namespace NifOsg
             if (mHasHerbalismLabel)
                 created->getOrCreateUserDataContainer()->addDescription(Constants::HerbalismLabel);
 
+            // When dealing with stencil buffer, draw order is especially sensitive. Make sure such objects are drawn with traversal order.
+            if (mHasStencilProperty)
+                created->getOrCreateStateSet()->setRenderBinDetails(2, "TraversalOrderBin");
+
             // Attach particle emitters to their nodes which should all be loaded by now.
             handleQueuedParticleEmitters(created, nif);
 
@@ -334,6 +339,21 @@ namespace NifOsg
         void applyNodeProperties(const Nif::Node *nifNode, osg::Node *applyTo, SceneUtil::CompositeStateSetUpdater* composite, Resource::ImageManager* imageManager, std::vector<unsigned int>& boundTextures, int animflags)
         {
             const Nif::PropertyList& props = nifNode->props;
+
+            bool hasStencilProperty = false;
+
+            for (size_t i = 0; i <props.length(); ++i)
+            {
+                if (props[i].empty())
+                    continue;
+
+                if (props[i].getPtr()->recType == Nif::RC_NiStencilProperty)
+                {
+                    hasStencilProperty = true;
+                    break;
+                }
+            }
+
             for (size_t i = 0; i <props.length(); ++i)
             {
                 if (!props[i].empty())
@@ -350,14 +370,14 @@ namespace NifOsg
                         if (props[i].getPtr()->recIndex == mFirstRootTextureIndex)
                             applyTo->setUserValue("overrideFx", 1);
                     }
-                    handleProperty(props[i].getPtr(), applyTo, composite, imageManager, boundTextures, animflags);
+                    handleProperty(props[i].getPtr(), applyTo, composite, imageManager, boundTextures, animflags, hasStencilProperty);
                 }
             }
 
             auto geometry = dynamic_cast<const Nif::NiGeometry*>(nifNode);
             // NiGeometry's NiAlphaProperty doesn't get handled here because it's a drawable property
             if (geometry && !geometry->shaderprop.empty())
-                handleProperty(geometry->shaderprop.getPtr(), applyTo, composite, imageManager, boundTextures, animflags);
+                handleProperty(geometry->shaderprop.getPtr(), applyTo, composite, imageManager, boundTextures, animflags, hasStencilProperty);
         }
 
         static void setupController(const Nif::Controller* ctrl, SceneUtil::Controller* toSetup, int animflags)
@@ -1347,7 +1367,7 @@ namespace NifOsg
             case 4: return osg::Stencil::GREATER;
             case 5: return osg::Stencil::NOTEQUAL;
             case 6: return osg::Stencil::GEQUAL;
-            case 7: return osg::Stencil::NEVER; // NifSkope says this is GL_ALWAYS, but in MW it's GL_NEVER
+            case 7: return osg::Stencil::ALWAYS;
             default:
                 Log(Debug::Info) << "Unexpected stencil function: " << func << " in " << mFilename;
                 return osg::Stencil::NEVER;
@@ -1772,7 +1792,7 @@ namespace NifOsg
         }
 
         void handleProperty(const Nif::Property *property,
-                            osg::Node *node, SceneUtil::CompositeStateSetUpdater* composite, Resource::ImageManager* imageManager, std::vector<unsigned int>& boundTextures, int animflags)
+                            osg::Node *node, SceneUtil::CompositeStateSetUpdater* composite, Resource::ImageManager* imageManager, std::vector<unsigned int>& boundTextures, int animflags, bool hasStencilProperty)
         {
             switch (property->recType)
             {
@@ -1800,6 +1820,7 @@ namespace NifOsg
 
                 if (stencilprop->data.enabled != 0)
                 {
+                    mHasStencilProperty = true;
                     osg::ref_ptr<osg::Stencil> stencil = new osg::Stencil;
                     stencil->setFunction(getStencilFunction(stencilprop->data.compareFunc), stencilprop->data.stencilRef, stencilprop->data.stencilMask);
                     stencil->setStencilFailOperation(getStencilOperation(stencilprop->data.failAction));
@@ -1831,7 +1852,9 @@ namespace NifOsg
                 osg::ref_ptr<osg::Depth> depth = new osg::Depth;
                 // Depth write flag
                 depth->setWriteMask((zprop->flags>>1)&1);
-                // Morrowind ignores depth test function
+                // Morrowind ignores depth test function, unless a NiStencilProperty is present, in which case it uses a fixed depth function of GL_ALWAYS.
+                if (hasStencilProperty)
+                    depth->setFunction(osg::Depth::ALWAYS);
                 depth = shareAttribute(depth);
                 stateset->setAttributeAndModes(depth, osg::StateAttribute::ON);
                 break;
@@ -2049,7 +2072,10 @@ namespace NifOsg
 
                         bool noSort = (alphaprop->flags>>13)&1;
                         if (!noSort)
+                        {
                             node->getOrCreateStateSet()->setRenderingHint(osg::StateSet::TRANSPARENT_BIN);
+                            node->getOrCreateStateSet()->setNestRenderBins(false);
+                        }
                         else
                             node->getOrCreateStateSet()->setRenderBinToInherit();
                     }

components/sceneutil/rtt.cpp

@@ -50,7 +50,7 @@ namespace SceneUtil
 
     osg::Texture* RTTNode::getDepthTexture(osgUtil::CullVisitor* cv)
     {
-        return getViewDependentData(cv)->mCamera->getBufferAttachmentMap()[osg::Camera::DEPTH_BUFFER]._texture;
+        return getViewDependentData(cv)->mCamera->getBufferAttachmentMap()[osg::Camera::PACKED_DEPTH_STENCIL_BUFFER]._texture;
     }
 
     RTTNode::ViewDependentData* RTTNode::getViewDependentData(osgUtil::CullVisitor* cv)
@@ -67,7 +67,7 @@ namespace SceneUtil
             mViewDependentDataMap[cv]->mCamera = camera;
 
             camera->setRenderOrder(osg::Camera::PRE_RENDER, mRenderOrderNum);
-            camera->setClearMask(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
+            camera->setClearMask(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
             camera->setRenderTargetImplementation(osg::Camera::FRAME_BUFFER_OBJECT);
             camera->setViewport(0, 0, mTextureWidth, mTextureHeight);
             SceneUtil::setCameraClearDepth(camera);
@@ -88,18 +88,18 @@ namespace SceneUtil
                 SceneUtil::attachAlphaToCoverageFriendlyFramebufferToCamera(camera, osg::Camera::COLOR_BUFFER, colorBuffer);
             }
 
-            if (camera->getBufferAttachmentMap().count(osg::Camera::DEPTH_BUFFER) == 0)
+            if (camera->getBufferAttachmentMap().count(osg::Camera::PACKED_DEPTH_STENCIL_BUFFER) == 0)
             {
                 auto depthBuffer = new osg::Texture2D;
                 depthBuffer->setTextureSize(mTextureWidth, mTextureHeight);
-                depthBuffer->setSourceFormat(GL_DEPTH_COMPONENT);
+                depthBuffer->setSourceFormat(GL_DEPTH_STENCIL_EXT);
-                depthBuffer->setInternalFormat(GL_DEPTH_COMPONENT24);
+                depthBuffer->setInternalFormat(GL_DEPTH24_STENCIL8_EXT);
                 depthBuffer->setWrap(osg::Texture::WRAP_S, osg::Texture::CLAMP_TO_EDGE);
                 depthBuffer->setWrap(osg::Texture::WRAP_T, osg::Texture::CLAMP_TO_EDGE);
-                depthBuffer->setSourceType(GL_UNSIGNED_INT);
+                depthBuffer->setSourceType(GL_UNSIGNED_INT_24_8_EXT);
                 depthBuffer->setFilter(osg::Texture::MIN_FILTER, osg::Texture::LINEAR);
                 depthBuffer->setFilter(osg::Texture::MAG_FILTER, osg::Texture::LINEAR);
-                camera->attach(osg::Camera::DEPTH_BUFFER, depthBuffer);
+                camera->attach(osg::Camera::PACKED_DEPTH_STENCIL_BUFFER, depthBuffer);
             }
         }
 

components/sceneutil/rtt.hpp

@@ -27,7 +27,7 @@ namespace SceneUtil
     /// @par Camera settings should be effectuated by overriding the setDefaults() and apply() methods, following a pattern similar to SceneUtil::StateSetUpdater
     /// @par When using the RTT texture in your statesets, it is recommended to use SceneUtil::StateSetUpdater as a cull callback to handle this as the appropriate
     ///     textures can be retrieved during SceneUtil::StateSetUpdater::Apply()
-    /// @par For any of COLOR_BUFFER or DEPTH_BUFFER not added during setDefaults(), RTTNode will attach a default buffer. The default color buffer has an internal format of GL_RGB.
+    /// @par For any of COLOR_BUFFER or PACKED_DEPTH_STENCIL_BUFFER not added during setDefaults(), RTTNode will attach a default buffer. The default color buffer has an internal format of GL_RGB.
     ///     The default depth buffer has internal format GL_DEPTH_COMPONENT24, source format GL_DEPTH_COMPONENT, and source type GL_UNSIGNED_INT. Default wrap is CLAMP_TO_EDGE and filter LINEAR.
     class RTTNode : public osg::Node
     {