feat: add VRM model loading functionality

This commit introduces the core VRM loading capabilities for the 3D rendering system. It implements a dedicated VRMLoader class that can parse VRM files (which are binary glTF format) and extract essential data including meshes, materials, textures, and VRM-specific extension metadata. The implementation handles both binary and JSON formatted VRM files while supporting standard glTF accessor data extraction for vertices, normals, UV coordinates, and indices. This enables the application to load and render 3D models with proper material and texture support from VRM files.

src/rendering/__init__.py

@@ -0,0 +1 @@
+"""3D rendering components for Desktop Waifu"""

src/rendering/vrm_loader.py

@@ -0,0 +1,243 @@
+"""
+VRM Loader - Loads and parses VRM model files
+"""
+import os
+import base64
+from typing import Optional, Dict, List, Tuple
+import numpy as np
+from pygltflib import GLTF2
+from PIL import Image
+import io
+
+class VRMModel:
+    """Container for loaded VRM model data"""
+
+    def __init__(self):
+        self.meshes: List[Dict] = []
+        self.materials: List[Dict] = []
+        self.textures: List[Optional[Image.Image]] = []
+        self.nodes: List[Dict] = []
+        self.vrm_meta: Optional[Dict] = None
+        self.blend_shapes: Dict[str, List] = {}
+
+class VRMLoader:
+    """Loads VRM models using pygltflib"""
+
+    def __init__(self):
+        self.model: Optional[VRMModel] = None
+        self.gltf: Optional[GLTF2] = None
+
+    def load(self, file_path: str) -> VRMModel:
+        """Load VRM file and extract data"""
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"VRM file not found: {file_path}")
+
+        print(f"Loading VRM: {file_path}")
+
+        # VRM files are binary glTF (.glb) format
+        # Check file magic bytes to determine if binary
+        with open(file_path, 'rb') as f:
+            magic = f.read(4)
+
+        is_binary = magic == b'glTF'
+
+        # Load glTF/VRM file
+        if is_binary:
+            self.gltf = GLTF2().load_binary(file_path)
+        else:
+            self.gltf = GLTF2().load_json(file_path)
+
+        self.model = VRMModel()
+
+        # Extract VRM extension data
+        self._extract_vrm_extensions()
+
+        # Extract meshes
+        self._extract_meshes()
+
+        # Extract materials
+        self._extract_materials()
+
+        # Extract textures
+        self._extract_textures(file_path)
+
+        print(f"VRM loaded: {len(self.model.meshes)} meshes, {len(self.model.materials)} materials")
+
+        return self.model
+
+    def _extract_vrm_extensions(self):
+        """Extract VRM-specific extension data"""
+        if hasattr(self.gltf, 'extensions') and self.gltf.extensions:
+            if 'VRM' in self.gltf.extensions:
+                self.model.vrm_meta = self.gltf.extensions['VRM']
+                print("VRM extension found")
+
+    def _extract_meshes(self):
+        """Extract mesh data from glTF"""
+        if not self.gltf.meshes:
+            return
+
+        for mesh_idx, mesh in enumerate(self.gltf.meshes):
+            mesh_data = {
+                'name': mesh.name or f'Mesh_{mesh_idx}',
+                'primitives': []
+            }
+
+            for prim in mesh.primitives:
+                prim_data = self._extract_primitive(prim)
+                mesh_data['primitives'].append(prim_data)
+
+            self.model.meshes.append(mesh_data)
+
+    def _extract_primitive(self, primitive) -> Dict:
+        """Extract primitive (submesh) data"""
+        prim_data = {
+            'vertices': None,
+            'normals': None,
+            'uvs': None,
+            'indices': None,
+            'material_idx': primitive.material if primitive.material is not None else 0
+        }
+
+        # Get vertices (POSITION)
+        if primitive.attributes.POSITION is not None:
+            prim_data['vertices'] = self._get_accessor_data(primitive.attributes.POSITION)
+
+        # Get normals
+        if primitive.attributes.NORMAL is not None:
+            prim_data['normals'] = self._get_accessor_data(primitive.attributes.NORMAL)
+
+        # Get UVs (TEXCOORD_0)
+        if primitive.attributes.TEXCOORD_0 is not None:
+            prim_data['uvs'] = self._get_accessor_data(primitive.attributes.TEXCOORD_0)
+
+        # Get indices
+        if primitive.indices is not None:
+            prim_data['indices'] = self._get_accessor_data(primitive.indices)
+
+        return prim_data
+
+    def _get_accessor_data(self, accessor_idx: int) -> np.ndarray:
+        """Get data from glTF accessor"""
+        accessor = self.gltf.accessors[accessor_idx]
+        buffer_view = self.gltf.bufferViews[accessor.bufferView]
+        buffer = self.gltf.buffers[buffer_view.buffer]
+
+        # Get buffer data
+        if buffer.uri:
+            # External buffer (shouldn't happen with .vrm files)
+            raise NotImplementedError("External buffers not supported")
+        else:
+            # Embedded buffer
+            data = self.gltf.binary_blob()
+
+        # Calculate offset
+        offset = buffer_view.byteOffset + accessor.byteOffset if accessor.byteOffset else buffer_view.byteOffset
+
+        # Map component type to numpy dtype
+        component_types = {
+            5120: np.int8,
+            5121: np.uint8,
+            5122: np.int16,
+            5123: np.uint16,
+            5125: np.uint32,
+            5126: np.float32,
+        }
+
+        dtype = component_types.get(accessor.componentType)
+        if dtype is None:
+            raise ValueError(f"Unknown component type: {accessor.componentType}")
+
+        # Map accessor type to element count
+        type_sizes = {
+            'SCALAR': 1,
+            'VEC2': 2,
+            'VEC3': 3,
+            'VEC4': 4,
+            'MAT2': 4,
+            'MAT3': 9,
+            'MAT4': 16,
+        }
+
+        element_size = type_sizes.get(accessor.type, 1)
+
+        # Extract data
+        count = accessor.count * element_size
+        array = np.frombuffer(data, dtype=dtype, count=count, offset=offset)
+
+        # Reshape if needed
+        if element_size > 1:
+            array = array.reshape((accessor.count, element_size))
+
+        return array
+
+    def _extract_materials(self):
+        """Extract material data"""
+        if not self.gltf.materials:
+            # Create default material
+            self.model.materials.append({
+                'name': 'Default',
+                'base_color': [1.0, 1.0, 1.0, 1.0],
+                'texture_idx': None
+            })
+            return
+
+        for mat_idx, material in enumerate(self.gltf.materials):
+            mat_data = {
+                'name': material.name or f'Material_{mat_idx}',
+                'base_color': [1.0, 1.0, 1.0, 1.0],
+                'texture_idx': None
+            }
+
+            # Get base color
+            if material.pbrMetallicRoughness:
+                if material.pbrMetallicRoughness.baseColorFactor:
+                    mat_data['base_color'] = material.pbrMetallicRoughness.baseColorFactor
+
+                # Get base color texture
+                if material.pbrMetallicRoughness.baseColorTexture:
+                    mat_data['texture_idx'] = material.pbrMetallicRoughness.baseColorTexture.index
+
+            self.model.materials.append(mat_data)
+
+    def _extract_textures(self, vrm_path: str):
+        """Extract texture images"""
+        if not self.gltf.textures:
+            return
+
+        base_dir = os.path.dirname(vrm_path)
+
+        for tex_idx, texture in enumerate(self.gltf.textures):
+            try:
+                image = self.gltf.images[texture.source]
+
+                if image.uri:
+                    # External image file
+                    if image.uri.startswith('data:'):
+                        # Data URI
+                        header, data = image.uri.split(',', 1)
+                        img_data = base64.b64decode(data)
+                        img = Image.open(io.BytesIO(img_data))
+                    else:
+                        # File path
+                        img_path = os.path.join(base_dir, image.uri)
+                        img = Image.open(img_path)
+                else:
+                    # Embedded in bufferView
+                    buffer_view = self.gltf.bufferViews[image.bufferView]
+                    data = self.gltf.binary_blob()
+                    offset = buffer_view.byteOffset
+                    length = buffer_view.byteLength
+                    img_data = data[offset:offset+length]
+                    img = Image.open(io.BytesIO(img_data))
+
+                # Convert to RGBA if needed
+                if img.mode != 'RGBA':
+                    img = img.convert('RGBA')
+
+                self.model.textures.append(img)
+                print(f"Loaded texture {tex_idx}: {img.size}")
+
+            except Exception as e:
+                print(f"Failed to load texture {tex_idx}: {e}")
+                self.model.textures.append(None)

src/ui/vrm_widget.py

@@ -1,6 +1,8 @@
 """
 VRM Widget - OpenGL widget for rendering VRM models
 """
+import os
+from typing import Optional
 from PyQt6.QtOpenGLWidgets import QOpenGLWidget
 from PyQt6.QtCore import Qt, QTimer
 from PyQt6.QtGui import QSurfaceFormat
@@ -9,6 +11,7 @@ from OpenGL.GLU import *
 import numpy as np
 
 from src.core.state_manager import StateManager, EmotionState
+from src.rendering.vrm_loader import VRMLoader, VRMModel
 
 class VRMWidget(QOpenGLWidget):
     """OpenGL widget for rendering VRM character model"""
@@ -22,9 +25,13 @@ class VRMWidget(QOpenGLWidget):
 
         super().__init__(parent)
         self.state_manager = state_manager
-        self.vrm_model = None
+        self.vrm_model: Optional[VRMModel] = None
         self.current_emotion = EmotionState.NEUTRAL
 
+        # OpenGL resources
+        self.textures = []
+        self.display_lists = []
+
         # Setup state change listener
         self.state_manager.register_listener('emotion_change', self.on_emotion_change)
 
@@ -38,18 +45,29 @@ class VRMWidget(QOpenGLWidget):
         # Enable depth testing
         glEnable(GL_DEPTH_TEST)
 
+        # Disable backface culling (VRM models sometimes have inverted faces)
+        glDisable(GL_CULL_FACE)
+
         # Enable blending for transparency
         glEnable(GL_BLEND)
         glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
 
+        # Enable lighting for basic shading
+        glEnable(GL_LIGHTING)
+        glEnable(GL_LIGHT0)
+        glEnable(GL_COLOR_MATERIAL)
+        glColorMaterial(GL_FRONT_AND_BACK, GL_AMBIENT_AND_DIFFUSE)
+
+        # Set up basic light (brighter for better visibility)
+        glLightfv(GL_LIGHT0, GL_POSITION, [0.0, 1.0, 2.0, 0.0])
+        glLightfv(GL_LIGHT0, GL_AMBIENT, [0.5, 0.5, 0.5, 1.0])
+        glLightfv(GL_LIGHT0, GL_DIFFUSE, [1.0, 1.0, 1.0, 1.0])
+
         # Set clear color (transparent)
         glClearColor(0.0, 0.0, 0.0, 0.0)
 
-        # TODO: Load VRM model
+        # Load VRM model
-        # self.load_vrm_model()
+        self._load_default_model()
-
-        # TODO: Setup shaders (MToon shader)
-        # self.setup_shaders()
 
         print("OpenGL initialized")
 
@@ -73,12 +91,18 @@ class VRMWidget(QOpenGLWidget):
         # Reset modelview matrix
         glLoadIdentity()
 
-        # Move camera back
+        # Move camera back and adjust vertical position to show full body
-        glTranslatef(0.0, -0.5, -3.0)
+        glTranslatef(0.0, -0.5, -2.2)
 
-        # TODO: Render VRM model
+        # Render VRM model if loaded, otherwise show placeholder
-        # For now, render a placeholder
+        if self.vrm_model:
-        self.render_placeholder()
+            # Rotate model to face camera (VRM models typically face +Z, we need -Z)
+            glRotatef(180, 0, 1, 0)
+            # Scale down VRM models (they're usually in meters, about 1.6 units tall)
+            glScalef(0.85, 0.85, 0.85)
+            self.render_vrm()
+        else:
+            self.render_placeholder()
 
     def render_placeholder(self):
         """Render a placeholder (cube) until VRM is loaded"""
@@ -144,8 +168,135 @@ class VRMWidget(QOpenGLWidget):
         # TODO: Update blend shapes/expressions based on emotion
         print(f"VRM Widget: Emotion changed to {new_emotion.value}")
 
+    def _load_default_model(self):
+        """Load the default VRM model from models folder"""
+        models_dir = os.path.join(os.getcwd(), 'models')
+
+        # Look for .vrm files in models directory
+        if os.path.exists(models_dir):
+            vrm_files = [f for f in os.listdir(models_dir) if f.endswith('.vrm')]
+            if vrm_files:
+                model_path = os.path.join(models_dir, vrm_files[0])
+                self.load_vrm_model(model_path)
+            else:
+                print("No VRM model found in models/ folder")
+        else:
+            print("models/ directory not found")
+
     def load_vrm_model(self, model_path: str):
         """Load VRM model from file"""
-        # TODO: Implement VRM loading using pygltflib
+        try:
-        print(f"Loading VRM model from {model_path}")
+            print(f"Loading VRM model from {model_path}")
-        pass
+            loader = VRMLoader()
+            self.vrm_model = loader.load(model_path)
+
+            # Create OpenGL textures
+            self._create_textures()
+
+            print(f"VRM model loaded successfully: {len(self.vrm_model.meshes)} meshes")
+
+        except Exception as e:
+            print(f"Failed to load VRM model: {e}")
+            import traceback
+            traceback.print_exc()
+
+    def _create_textures(self):
+        """Create OpenGL textures from loaded images"""
+        if not self.vrm_model or not self.vrm_model.textures:
+            print("No textures to create")
+            return
+
+        print(f"Creating {len(self.vrm_model.textures)} OpenGL textures...")
+        for idx, img in enumerate(self.vrm_model.textures):
+            if img is None:
+                print(f"  Texture {idx}: None (skipped)")
+                self.textures.append(None)
+                continue
+
+            tex_id = glGenTextures(1)
+            glBindTexture(GL_TEXTURE_2D, tex_id)
+
+            # Convert PIL image to bytes
+            img_data = img.tobytes()
+            width, height = img.size
+
+            # Upload texture
+            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, img_data)
+
+            # Set texture parameters
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
+            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
+
+            self.textures.append(tex_id)
+            print(f"  Texture {idx}: {width}x{height}, ID={tex_id}")
+
+    def render_vrm(self):
+        """Render the loaded VRM model"""
+        if not self.vrm_model:
+            return
+
+        # Enable texturing
+        glEnable(GL_TEXTURE_2D)
+
+        # Set texture environment to modulate (combine texture with lighting)
+        glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE)
+
+        # Render each mesh
+        for mesh in self.vrm_model.meshes:
+            for primitive in mesh['primitives']:
+                self._render_primitive(primitive)
+
+        glDisable(GL_TEXTURE_2D)
+
+    def _render_primitive(self, primitive):
+        """Render a single primitive (submesh)"""
+        vertices = primitive.get('vertices')
+        normals = primitive.get('normals')
+        uvs = primitive.get('uvs')
+        indices = primitive.get('indices')
+        material_idx = primitive.get('material_idx', 0)
+
+        if vertices is None:
+            return
+
+        # Get material
+        has_texture = False
+        if material_idx < len(self.vrm_model.materials):
+            material = self.vrm_model.materials[material_idx]
+
+            # Bind texture if available
+            tex_idx = material.get('texture_idx')
+            if tex_idx is not None and tex_idx < len(self.textures) and self.textures[tex_idx]:
+                glBindTexture(GL_TEXTURE_2D, self.textures[tex_idx])
+                has_texture = True
+                # Use white color when textured so texture shows through properly
+                glColor4f(1.0, 1.0, 1.0, 1.0)
+            else:
+                glBindTexture(GL_TEXTURE_2D, 0)
+                # Use material base color when no texture
+                color = material.get('base_color', [1.0, 1.0, 1.0, 1.0])
+                glColor4fv(color)
+
+        # Render with immediate mode
+        if indices is not None:
+            glBegin(GL_TRIANGLES)
+            for idx in indices:
+                if normals is not None and idx < len(normals):
+                    glNormal3fv(normals[idx])
+                if uvs is not None and idx < len(uvs):
+                    glTexCoord2fv(uvs[idx])
+                if idx < len(vertices):
+                    glVertex3fv(vertices[idx])
+            glEnd()
+        else:
+            # No indices, render directly
+            glBegin(GL_TRIANGLES)
+            for i in range(len(vertices)):
+                if normals is not None and i < len(normals):
+                    glNormal3fv(normals[i])
+                if uvs is not None and i < len(uvs):
+                    glTexCoord2fv(uvs[i])
+                glVertex3fv(vertices[i])
+            glEnd()