implement stencil test in Level1 and integrate it in render pass

libopenage/presenter/presenter.cpp

@@ -20,8 +20,6 @@
 #include "renderer/camera/camera.h"
 #include "renderer/gui/gui.h"
 #include "renderer/gui/integration/public/gui_application_with_logger.h"
-#include "renderer/opengl/context.h"
-#include "renderer/opengl/lookup.h"
 #include "renderer/render_factory.h"
 #include "renderer/render_pass.h"
 #include "renderer/render_target.h"
@@ -167,10 +165,19 @@ void Presenter::init_graphics(const renderer::window_settings &window_settings)
 		this->asset_manager,
 		this->time_loop->get_clock());
 	this->render_passes.push_back(this->hud_renderer->get_render_pass());
+
+	for (auto& render_pass : render_passes) {
+		render_pass->set_stencil_state(renderer::StencilState::USE_STENCIL_TEST);
+	}
 
 	this->init_gui();
 	this->init_final_render_pass();
 
+	// set passes indices
+	this->index_gui_stencil_pass = this->render_passes.size() - 3;
+	this->index_gui_render_pass = this->render_passes.size() - 2;
+	this->index_final_render_pass = this->render_passes.size() - 1;
+
 	if (this->simulation) {
 		auto render_factory = std::make_shared<renderer::RenderFactory>(this->terrain_renderer, this->world_renderer);
 		this->simulation->attach_renderer(render_factory);
@@ -213,7 +220,10 @@ void Presenter::init_gui() {
 		qml_root,      // directory to watch for qml file changes
 		qml_assets,    // qml data: Engine *, the data directory, ...
 		this->renderer // openage renderer
-	);
+	);	
+
+	auto stencil_pass = this->gui->get_stencil_pass();
+	this->render_passes.push_back(stencil_pass);
 
 	auto gui_pass = this->gui->get_render_pass();
 	this->render_passes.push_back(gui_pass);
@@ -309,57 +319,24 @@ void Presenter::init_final_render_pass() {
 	});
 }
 
-void Presenter::init_stencil_test() {
-	glEnable(GL_STENCIL_TEST);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-	glClear(GL_STENCIL_BUFFER_BIT);
-}
-
-void Presenter::enable_stencil_for_gui_mask() {
-	// Replace stencil value with 1 when depth test passes
-	glStencilFunc(GL_ALWAYS, 1, 0xFF);
-	glStencilMask(0xFF);
-	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-}
-
-void Presenter::enable_stencil_for_world() {
-	// Only pass if stencil value is not 1
-	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
-	glStencilMask(0x00);
-	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-}
-
-void Presenter::disable_stencil() {
-	glDisable(GL_STENCIL_TEST);
-	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-}
-
 void Presenter::render() {
 	// TODO: Pass current time to update() instead of fetching it in renderer
 	this->camera_manager->update();
 	this->terrain_renderer->update();
 	this->world_renderer->update();
 	this->hud_renderer->update();
 
-	this->init_stencil_test();
-
-	// First Pass: Render GUI to stencil buffer
-	this->enable_stencil_for_gui_mask();
 	this->gui->render();
+	this->renderer->render(this->render_passes[this->index_gui_stencil_pass]);
 
-	// Second Pass: Render game world with stencil buffer
-	this->enable_stencil_for_world();
-	for (size_t i = 0; i < this->render_passes.size() - 2; ++i) {
+	for (size_t i = 0; i < this->index_gui_stencil_pass; ++i) {
 		this->renderer->render(this->render_passes[i]);
 	}
 
-	// Third Pass: Render GUI to screen
-	this->disable_stencil();
 	this->gui->render();
-	this->renderer->render(this->render_passes[this->render_passes.size() - 2]);
+	this->renderer->render(this->render_passes[this->index_gui_render_pass]);
 
-	// Fourth Pass: Render screen to window
-	this->renderer->render(this->render_passes.back());
+	this->renderer->render(this->render_passes[this->index_final_render_pass]);
 }
 
 } // namespace openage::presenter

libopenage/presenter/presenter.h

@@ -142,11 +142,6 @@ class Presenter {
 
 	// void init_audio();
 
-	void init_stencil_test();
-	void enable_stencil_for_gui_mask();
-	void enable_stencil_for_world();
-	void disable_stencil();
-
 	/**
 	 * Render all configured render passes in sequence.
 	 */
@@ -235,6 +230,13 @@ class Presenter {
 	 * Input manager.
 	 */
 	std::shared_ptr<input::InputManager> input_manager;
+
+	/**
+	 * Pass indices.
+	 */
+	size_t index_gui_stencil_pass;
+	size_t index_gui_render_pass;
+	size_t index_final_render_pass;
 };
 
 } // namespace presenter

libopenage/renderer/gui/gui.cpp

@@ -62,6 +62,10 @@ std::shared_ptr<renderer::RenderPass> GUI::get_render_pass() const {
 	return this->render_pass;
 }
 
+std::shared_ptr<renderer::RenderPass> GUI::get_stencil_pass() const {
+	return this->stencil_pass;
+}
+
 void GUI::initialize_render_pass(size_t width,
                                  size_t height,
                                  const util::Path &shaderdir) {
@@ -98,6 +102,10 @@ void GUI::initialize_render_pass(size_t width,
 	// TODO: Rendering into the FBO is a bit redundant right now because we
 	// just copy the GUI texture into the output texture
 	this->render_pass = renderer->add_render_pass({display_obj}, fbo);
+
+	auto stencil_pass = renderer->add_render_pass({display_obj}, fbo);
+	stencil_pass->set_stencil_state(StencilState::WRITE_STENCIL_MASK);
+	this->stencil_pass = stencil_pass;
 }
 
 

libopenage/renderer/gui/gui.h

@@ -63,6 +63,13 @@ class GUI {
 	 */
 	std::shared_ptr<renderer::RenderPass> get_render_pass() const;
 
+	/**
+	 * Get the stencil render pass of the GUI.
+	 * 
+	 * @return stencil render pass of the GUI	
+	 */
+	std::shared_ptr<renderer::RenderPass> get_stencil_pass() const;
+
 	/**
 	 * Render the GUI texture.
 	 */
@@ -139,6 +146,11 @@ class GUI {
 	 * this pass via a \p renderer::resources::Renderable.
 	 */
 	std::shared_ptr<renderer::RenderPass> render_pass;
+
+	/**
+	 * Render pass for the stencil mask of the GUI.
+	 */
+	std::shared_ptr<renderer::RenderPass> stencil_pass;
 };
 
 } // namespace gui

libopenage/renderer/opengl/renderer.cpp

@@ -162,6 +162,30 @@ void GlRenderer::optimize(const std::shared_ptr<GlRenderPass> &pass) {
 	}
 }
 
+void GlRenderer::setupStencilWrite() {
+	glEnable(GL_STENCIL_TEST);
+	glStencilFunc(GL_ALWAYS, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+	glStencilMask(0xFF);
+	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+	glDepthMask(GL_FALSE);
+}
+
+void GlRenderer::setupStencilTest() {
+	glEnable(GL_STENCIL_TEST);
+	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+	glStencilMask(0x00);
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	glDepthMask(GL_TRUE);
+}
+
+void GlRenderer::disableStencilTest() {
+	glDisable(GL_STENCIL_TEST);
+	glStencilMask(0xFF);
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	glDepthMask(GL_TRUE);
+}
+
 void GlRenderer::check_error() {
 	// thanks for the global state, opengl!
 	GlContext::check_error();
@@ -176,7 +200,12 @@ void GlRenderer::render(const std::shared_ptr<RenderPass> &pass) {
 	// see https://www.khronos.org/opengl/wiki/Vertex_Rendering#Causes_of_rendering_failure
 	shared_quad_vao->bind();
 
-	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+	// ensure that an (empty) VAO is bound before rendering geometry
+	// a bound VAO is required to render bufferless quad geometries by OpenGL
+	// see https://www.khronos.org/opengl/wiki/Vertex_Rendering#Causes_of_rendering_failure
+	shared_quad_vao->bind();
+
+	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 
 	// TODO: Option for face culling
 	// glEnable(GL_CULL_FACE);
@@ -198,6 +227,20 @@ void GlRenderer::render(const std::shared_ptr<RenderPass> &pass) {
 			glClear(GL_DEPTH_BUFFER_BIT);
 		}
 
+		switch (layer.stencil_state) {
+		case StencilState::WRITE_STENCIL_MASK:
+			setupStencilWrite();
+			break;
+
+		case StencilState::USE_STENCIL_TEST:
+			setupStencilTest();
+			break;
+
+		case StencilState::DISABLE_STENCIL:
+			disableStencilTest();
+			break;
+		}
+
 		for (auto const &obj : objects) {
 			if (obj.alpha_blending) {
 				glEnable(GL_BLEND);

libopenage/renderer/opengl/renderer.h

@@ -63,6 +63,12 @@ class GlRenderer final : public Renderer {
 	/// Optimize the render pass by reordering stuff
 	static void optimize(const std::shared_ptr<GlRenderPass> &pass);
 
+	void setupStencilWrite();
+
+	void setupStencilTest();
+
+	void disableStencilTest();
+
 	/// The GL context.
 	std::shared_ptr<GlContext> gl_context;
 

libopenage/renderer/render_pass.cpp

@@ -83,7 +83,7 @@ void RenderPass::add_renderables(Renderable &&renderable, int64_t priority) {
 	this->add_renderables(std::vector<Renderable>{std::move(renderable)}, priority);
 }
 
-void RenderPass::add_layer(int64_t priority, bool clear_depth) {
+void RenderPass::add_layer(int64_t priority, bool clear_depth, StencilState stencil_state) {
 	size_t layer_index = 0;
 	for (const auto &layer : this->layers) {
 		if (layer.priority > priority) {
@@ -92,14 +92,20 @@ void RenderPass::add_layer(int64_t priority, bool clear_depth) {
 		layer_index++;
 	}
 
-	this->add_layer(layer_index, priority, clear_depth);
+	this->add_layer(layer_index, priority, clear_depth, stencil_state);
 }
 
-void RenderPass::add_layer(size_t index, int64_t priority, bool clear_depth) {
-	this->layers.insert(this->layers.begin() + index, Layer{priority, clear_depth});
+void RenderPass::add_layer(size_t index, int64_t priority, bool clear_depth, StencilState stencil_state) {
+	this->layers.insert(this->layers.begin() + index, Layer{priority, clear_depth, stencil_state});
 	this->renderables.insert(this->renderables.begin() + index, std::vector<Renderable>{});
 }
 
+void RenderPass::set_stencil_state(StencilState state) {
+	for (auto &layer : this->layers) {
+		layer.stencil_state = state;
+	}
+}
+
 void RenderPass::clear_renderables() {
 	// Keep layer definitions, but reset the length of each layer to 0
 	for (size_t i = 0; i < this->layers.size(); i++) {

libopenage/renderer/render_pass.h

@@ -14,6 +14,18 @@ namespace openage {
 namespace renderer {
 class RenderTarget;
 
+/**
+ * Stencil states for the render pass.
+ */
+enum class StencilState {
+	/// State for writing GUI elements to stencil buffer.
+	WRITE_STENCIL_MASK, 
+	/// State for using the mask when rendering scene.
+	USE_STENCIL_TEST, 
+	/// State for normal rendering (GUI rendering).
+	DISABLE_STENCIL,  
+};
+
 /**
  * Defines a layer in the render pass. A layer is a slice of the renderables
  * that have the same priority. Each layer can have its own settings.
@@ -27,6 +39,8 @@ struct Layer {
 	int64_t priority;
 	/// Whether to clear the depth buffer before rendering this layer.
 	bool clear_depth = true;
+	/// The state of the stencil buffer for the render pass.
+	StencilState stencil_state;
 };
 
 /**
@@ -95,8 +109,17 @@ class RenderPass {
 	 *
 	 * @param priority Priority of the layer. Layers with higher priority are drawn first.
 	 * @param clear_depth If true clears the depth buffer before rendering this layer.
+     * @param stencil_state State of the stencil buffer, using to do stencil test. 
 	 */
-	void add_layer(int64_t priority, bool clear_depth = true);
+	void add_layer(int64_t priority, bool clear_depth = true, 
+				   StencilState stencil_state = StencilState::DISABLE_STENCIL);
+
+	/**
+	 * Set the stencil state for the render pass.
+	 *
+	 * @param state The new stencil state.
+	 */
+	void set_stencil_state(StencilState state);
 
 	/**
 	 * Clear the list of renderables
@@ -137,9 +160,11 @@ class RenderPass {
 	 *
 	 * @param index Index in \p layers member to insert the new layer.
 	 * @param priority Priority of the layer. Layers with higher priority are drawn first.
-	 * @param clear_depth If true clears the depth buffer before rendering this layer.
+	 * @param clear_depth If true clears the depth buffer before rendering this layer. 
+     * @param stencil_state State of the stencil buffer, using to do stencil test. 
 	 */
-	void add_layer(size_t index, int64_t priority, bool clear_depth = true);
+	void add_layer(size_t index, int64_t priority, bool clear_depth = true, 
+				   StencilState stencil_state = StencilState::DISABLE_STENCIL);
 
 	/**
 	 * Render target to write to.