Fix the look of EFR Boats under batching (#2186)

glsm/src/main/java/com/gtnewhorizons/angelica/glsm/GLStateManager.java

@@ -1539,6 +1539,10 @@ public class GLStateManager {
         return ctx().depthState.isEffectiveMaskEnabled();
     }
 
+    public static ColorMask getEffectiveColorMask(ColorMask out) {
+        return ctx().colorMask.readEffective(out);
+    }
+
     public static boolean isEffectiveAlphaTestEnabled() {
         return ctx().alphaTest.isEffectivelyEnabled();
     }

glsm/src/main/java/com/gtnewhorizons/angelica/glsm/stacks/ColorMaskStack.java

@@ -39,6 +39,12 @@ public final class ColorMaskStack extends ColorMask implements CowStateStack<Col
         }
     }
 
+    public ColorMask readEffective(ColorMask out) {
+        out.set(this);
+        VanillaStateLayer.capture(vanillaLayer, out);
+        return out;
+    }
+
     @Override
     public boolean topSlotChanged() {
         return !cow.isEmpty() && !sameAs(stack[cow.top()]);

src/main/java/com/gtnewhorizons/angelica/rendering/tesr/EntityMaterials.java

@@ -4,6 +4,7 @@ import com.gtnewhorizons.angelica.api.tesr.TesrMaterial;
 import com.gtnewhorizons.angelica.glsm.GLStateManager;
 import com.gtnewhorizons.angelica.glsm.states.AlphaState;
 import com.gtnewhorizons.angelica.glsm.states.BlendState;
+import com.gtnewhorizons.angelica.glsm.states.ColorMask;
 import org.lwjgl.opengl.GL11;
 
 public final class EntityMaterials {
@@ -26,19 +27,27 @@ public final class EntityMaterials {
     public static final TesrMaterial DROPPED_ITEM_TRANSLUCENT = TesrMaterial.builder().translucent().cutout(0.1f).unfilteredAtlas().stream().build();
     public static final TesrMaterial HELD_BLOCK_CUTOUT = TesrMaterial.builder().cutout(0.1f).stream().build();
     public static final TesrMaterial HELD_BLOCK_TRANSLUCENT = TesrMaterial.builder().translucent().cutout(0.1f).stream().build();
+    static final TesrMaterial DEPTH_ONLY = TesrMaterial.builder().depthOnly().stream().build();
+    static final TesrMaterial DEPTH_ONLY_CUTOUT = TesrMaterial.builder().depthOnly().cutout(0.1f).stream().build();
     static final TesrMaterial OVERLAY = TesrMaterial.builder().translucent().depthEqual().stream().build();
     public static final TesrMaterial GLINT = TesrMaterial.builder().glint().depthEqual().noDepthWrite().unlit().stream().special(TesrMaterial.SpecialRender.GLINT).build();
     public static final TesrMaterial ITEM_GLINT = TesrMaterial.builder().glint().depthEqual().noDepthWrite().unlit().stream().special(TesrMaterial.SpecialRender.GLINT).build();
 
     private static final BlendState effectiveBlend = new BlendState();
     private static final AlphaState effectiveAlpha = new AlphaState();
+    private static final ColorMask effectiveColorMask = new ColorMask();
 
     private EntityMaterials() {}
 
     static TesrMaterial fromCurrentState(boolean texIdentity) {
         final boolean textured = GLStateManager.getTextures().getTextureUnitStates(0).isEnabled();
-        final BlendState blend = GLStateManager.getEffectiveBlendState(effectiveBlend);
         final AlphaState alpha = GLStateManager.getEffectiveAlphaState(effectiveAlpha);
+        final ColorMask colorMask = GLStateManager.getEffectiveColorMask(effectiveColorMask);
+        if (!colorMask.red && !colorMask.green && !colorMask.blue && !colorMask.alpha) {
+            return depthOnlyFromState(textured, GLStateManager.isEffectiveAlphaTestEnabled(), alpha.getFunction(), alpha.getReference(),
+                GLStateManager.getDepthState().getFunc(), GLStateManager.isEffectiveDepthMaskEnabled());
+        }
+        final BlendState blend = GLStateManager.getEffectiveBlendState(effectiveBlend);
         return fromState(textured, textured && !texIdentity, GLStateManager.isEffectiveBlendEnabled(), blend.getSrcRgb(), blend.getDstRgb(),
             GLStateManager.isEffectiveAlphaTestEnabled(), alpha.getFunction(), alpha.getReference(), GLStateManager.getDepthState().getFunc(),
             GLStateManager.isEffectiveDepthMaskEnabled());
@@ -64,6 +73,14 @@ public final class EntityMaterials {
         return unfilteredAtlas ? DROPPED_ITEM_CUTOUT : HELD_BLOCK_CUTOUT;
     }
 
+    static TesrMaterial depthOnlyFromState(boolean textured, boolean alphaTest, int alphaFunc, float alphaRef, int depthFunc, boolean depthMask) {
+        if (!textured || !depthMask) return null;
+        if (depthFunc != GL11.GL_LEQUAL && depthFunc != GL11.GL_LESS) return null;
+        if (!alphaTest) return DEPTH_ONLY;
+        if (alphaFunc == GL11.GL_GREATER && Math.abs(alphaRef - 0.1f) < 1e-4f) return DEPTH_ONLY_CUTOUT;
+        return null;
+    }
+
     static TesrMaterial fromState(boolean textured, boolean blend, int srcRgb, int dstRgb, boolean alphaTest, int alphaFunc, float alphaRef, int depthFunc, boolean depthMask) {
         return fromState(textured, false, blend, srcRgb, dstRgb, alphaTest, alphaFunc, alphaRef, depthFunc, depthMask);
     }

src/test/java/com/gtnewhorizons/angelica/rendering/tesr/EntityMaterialsTest.java

@@ -3,8 +3,10 @@ package com.gtnewhorizons.angelica.rendering.tesr;
 import org.junit.jupiter.api.Test;
 import org.lwjgl.opengl.GL11;
 
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 class EntityMaterialsTest {
 
@@ -104,6 +106,22 @@ class EntityMaterialsTest {
         assertNull(EntityMaterials.itemFromState(true, true, GL11.GL_ONE, GL11.GL_ONE, true, GL11.GL_GREATER, 0.1f, GL11.GL_LEQUAL, true), "additive blend has no item material");
     }
 
+    @Test
+    void colorMaskedOffMapsToDepthOnly() {
+        assertSame(EntityMaterials.DEPTH_ONLY_CUTOUT, EntityMaterials.depthOnlyFromState(true, true, GL11.GL_GREATER, 0.1f, GL11.GL_LEQUAL, true));
+        assertSame(EntityMaterials.DEPTH_ONLY, EntityMaterials.depthOnlyFromState(true, false, GL11.GL_ALWAYS, 0f, GL11.GL_LEQUAL, true));
+        assertFalse(DrawState.forMaterial(EntityMaterials.DEPTH_ONLY, DrawState.CULL_BACK, true, true, 0f, 0f).isColorWrite(), "depth-only must replay with color writes off");
+        assertTrue(DrawState.forMaterial(EntityMaterials.DEPTH_ONLY, DrawState.CULL_BACK, true, true, 0f, 0f).isDepthWrite(), "depth-only must replay with depth writes on");
+    }
+
+    @Test
+    void unrepresentableDepthOnlyStaysLive() {
+        assertNull(EntityMaterials.depthOnlyFromState(true, true, GL11.GL_GREATER, 0.5f, GL11.GL_LEQUAL, true), "nonstandard alpha ref must stay live");
+        assertNull(EntityMaterials.depthOnlyFromState(true, false, GL11.GL_ALWAYS, 0f, GL11.GL_LEQUAL, false), "no color and no depth writes must stay live");
+        assertNull(EntityMaterials.depthOnlyFromState(true, false, GL11.GL_ALWAYS, 0f, GL11.GL_EQUAL, true), "non-LEQUAL/LESS depth func must stay live");
+        assertNull(EntityMaterials.depthOnlyFromState(false, false, GL11.GL_ALWAYS, 0f, GL11.GL_LEQUAL, true), "texture off must stay live");
+    }
+
     @Test
     void unrecognizedStatesFallBackToLive() {
         assertNull(EntityMaterials.fromState(true, true, GL11.GL_SRC_COLOR, GL11.GL_ONE, false, GL11.GL_ALWAYS, 0f, GL11.GL_EQUAL, false), () -> "glint blend at EQUAL must stay live");

src/test/java/com/gtnewhorizons/angelica/rendering/tesr/TesrLayerStateTest.java

@@ -52,6 +52,7 @@ class TesrLayerStateTest {
         GLStateManager.enableDepthTest();
         GLStateManager.glDepthFunc(GL11.GL_LEQUAL);
         GLStateManager.glDepthMask(true);
+        GLStateManager.glColorMask(true, true, true, true);
         GLStateManager.disableBlend();
         GLStateManager.enableCull();
         GLStateManager.enableLighting();
@@ -64,6 +65,7 @@ class TesrLayerStateTest {
         GLStateManager.getAlphaTest().setVanillaLayer(null);
         GLStateManager.getAlphaState().setVanillaLayer(null);
         GLStateManager.getDepthState().setVanillaLayer(null);
+        GLStateManager.getColorMask().setVanillaLayer(null);
 
         ModelPartBatcher.INSTANCE.clear();
         GLStateManager.glActiveTexture(GL13.GL_TEXTURE1);
@@ -520,6 +522,7 @@ class TesrLayerStateTest {
     @Test
     void effectiveDepthMaskSurvivesAnotherPassHoldingItOff() {
         GLStateManager.getDepthState().setVanillaLayer(DepthColorStorage.DEPTH_LAYER);
+        GLStateManager.getColorMask().setVanillaLayer(DepthColorStorage.COLOR_LAYER);
 
         GLStateManager.glEnable(GL11.GL_TEXTURE_2D);
         GLStateManager.enableBlend();
@@ -532,6 +535,25 @@ class TesrLayerStateTest {
         assertSame(EntityMaterials.TRANSLUCENT_DEPTH_WRITE, EntityMaterials.fromCurrentState(true), "raw depth mask held off by another pass; the effective vanilla mask must pick the depth-write variant");
     }
 
+    @Test
+    void effectiveColorMaskPicksDepthOnlyWhileAnotherPassHoldsIt() {
+        GLStateManager.getDepthState().setVanillaLayer(DepthColorStorage.DEPTH_LAYER);
+        GLStateManager.getColorMask().setVanillaLayer(DepthColorStorage.COLOR_LAYER);
+
+        GLStateManager.glEnable(GL11.GL_TEXTURE_2D);
+        GLStateManager.disableBlend();
+        GLStateManager.enableAlphaTest();
+        GLStateManager.glAlphaFunc(GL11.GL_GREATER, 0.1f);
+        GLStateManager.glDepthFunc(GL11.GL_LEQUAL);
+        GLStateManager.glDepthMask(true);
+
+        DepthColorStorage.disableDepthColor();
+        assertSame(EntityMaterials.CUTOUT, EntityMaterials.fromCurrentState(true), "raw color mask held off by another pass is not a depth-only draw");
+
+        DepthColorStorage.deferColorMask(false, false, false, false);
+        assertSame(EntityMaterials.DEPTH_ONLY_CUTOUT, EntityMaterials.fromCurrentState(true), "vanilla color mask off while held must pick depth-only");
+    }
+
     private static boolean unitTexIdentity() {
         return Reflect.<Boolean>invoke(ModelPartBatcher.INSTANCE, "unitTexIdentity", new Class<?>[0]);
     }