diff --git a/src/rendering/__init__.py b/src/rendering/__init__.py new file mode 100644 index 0000000..a0f7745 --- /dev/null +++ b/src/rendering/__init__.py @@ -0,0 +1 @@ +"""3D rendering components for Desktop Waifu""" diff --git a/src/rendering/vrm_loader.py b/src/rendering/vrm_loader.py new file mode 100644 index 0000000..0235672 --- /dev/null +++ b/src/rendering/vrm_loader.py @@ -0,0 +1,243 @@ +""" +VRM Loader - Loads and parses VRM model files +""" +import os +import base64 +from typing import Optional, Dict, List, Tuple +import numpy as np +from pygltflib import GLTF2 +from PIL import Image +import io + +class VRMModel: + """Container for loaded VRM model data""" + + def __init__(self): + self.meshes: List[Dict] = [] + self.materials: List[Dict] = [] + self.textures: List[Optional[Image.Image]] = [] + self.nodes: List[Dict] = [] + self.vrm_meta: Optional[Dict] = None + self.blend_shapes: Dict[str, List] = {} + +class VRMLoader: + """Loads VRM models using pygltflib""" + + def __init__(self): + self.model: Optional[VRMModel] = None + self.gltf: Optional[GLTF2] = None + + def load(self, file_path: str) -> VRMModel: + """Load VRM file and extract data""" + if not os.path.exists(file_path): + raise FileNotFoundError(f"VRM file not found: {file_path}") + + print(f"Loading VRM: {file_path}") + + # VRM files are binary glTF (.glb) format + # Check file magic bytes to determine if binary + with open(file_path, 'rb') as f: + magic = f.read(4) + + is_binary = magic == b'glTF' + + # Load glTF/VRM file + if is_binary: + self.gltf = GLTF2().load_binary(file_path) + else: + self.gltf = GLTF2().load_json(file_path) + + self.model = VRMModel() + + # Extract VRM extension data + self._extract_vrm_extensions() + + # Extract meshes + self._extract_meshes() + + # Extract materials + self._extract_materials() + + # Extract textures + self._extract_textures(file_path) + + print(f"VRM loaded: {len(self.model.meshes)} meshes, {len(self.model.materials)} materials") + + return self.model + + def _extract_vrm_extensions(self): + """Extract VRM-specific extension data""" + if hasattr(self.gltf, 'extensions') and self.gltf.extensions: + if 'VRM' in self.gltf.extensions: + self.model.vrm_meta = self.gltf.extensions['VRM'] + print("VRM extension found") + + def _extract_meshes(self): + """Extract mesh data from glTF""" + if not self.gltf.meshes: + return + + for mesh_idx, mesh in enumerate(self.gltf.meshes): + mesh_data = { + 'name': mesh.name or f'Mesh_{mesh_idx}', + 'primitives': [] + } + + for prim in mesh.primitives: + prim_data = self._extract_primitive(prim) + mesh_data['primitives'].append(prim_data) + + self.model.meshes.append(mesh_data) + + def _extract_primitive(self, primitive) -> Dict: + """Extract primitive (submesh) data""" + prim_data = { + 'vertices': None, + 'normals': None, + 'uvs': None, + 'indices': None, + 'material_idx': primitive.material if primitive.material is not None else 0 + } + + # Get vertices (POSITION) + if primitive.attributes.POSITION is not None: + prim_data['vertices'] = self._get_accessor_data(primitive.attributes.POSITION) + + # Get normals + if primitive.attributes.NORMAL is not None: + prim_data['normals'] = self._get_accessor_data(primitive.attributes.NORMAL) + + # Get UVs (TEXCOORD_0) + if primitive.attributes.TEXCOORD_0 is not None: + prim_data['uvs'] = self._get_accessor_data(primitive.attributes.TEXCOORD_0) + + # Get indices + if primitive.indices is not None: + prim_data['indices'] = self._get_accessor_data(primitive.indices) + + return prim_data + + def _get_accessor_data(self, accessor_idx: int) -> np.ndarray: + """Get data from glTF accessor""" + accessor = self.gltf.accessors[accessor_idx] + buffer_view = self.gltf.bufferViews[accessor.bufferView] + buffer = self.gltf.buffers[buffer_view.buffer] + + # Get buffer data + if buffer.uri: + # External buffer (shouldn't happen with .vrm files) + raise NotImplementedError("External buffers not supported") + else: + # Embedded buffer + data = self.gltf.binary_blob() + + # Calculate offset + offset = buffer_view.byteOffset + accessor.byteOffset if accessor.byteOffset else buffer_view.byteOffset + + # Map component type to numpy dtype + component_types = { + 5120: np.int8, + 5121: np.uint8, + 5122: np.int16, + 5123: np.uint16, + 5125: np.uint32, + 5126: np.float32, + } + + dtype = component_types.get(accessor.componentType) + if dtype is None: + raise ValueError(f"Unknown component type: {accessor.componentType}") + + # Map accessor type to element count + type_sizes = { + 'SCALAR': 1, + 'VEC2': 2, + 'VEC3': 3, + 'VEC4': 4, + 'MAT2': 4, + 'MAT3': 9, + 'MAT4': 16, + } + + element_size = type_sizes.get(accessor.type, 1) + + # Extract data + count = accessor.count * element_size + array = np.frombuffer(data, dtype=dtype, count=count, offset=offset) + + # Reshape if needed + if element_size > 1: + array = array.reshape((accessor.count, element_size)) + + return array + + def _extract_materials(self): + """Extract material data""" + if not self.gltf.materials: + # Create default material + self.model.materials.append({ + 'name': 'Default', + 'base_color': [1.0, 1.0, 1.0, 1.0], + 'texture_idx': None + }) + return + + for mat_idx, material in enumerate(self.gltf.materials): + mat_data = { + 'name': material.name or f'Material_{mat_idx}', + 'base_color': [1.0, 1.0, 1.0, 1.0], + 'texture_idx': None + } + + # Get base color + if material.pbrMetallicRoughness: + if material.pbrMetallicRoughness.baseColorFactor: + mat_data['base_color'] = material.pbrMetallicRoughness.baseColorFactor + + # Get base color texture + if material.pbrMetallicRoughness.baseColorTexture: + mat_data['texture_idx'] = material.pbrMetallicRoughness.baseColorTexture.index + + self.model.materials.append(mat_data) + + def _extract_textures(self, vrm_path: str): + """Extract texture images""" + if not self.gltf.textures: + return + + base_dir = os.path.dirname(vrm_path) + + for tex_idx, texture in enumerate(self.gltf.textures): + try: + image = self.gltf.images[texture.source] + + if image.uri: + # External image file + if image.uri.startswith('data:'): + # Data URI + header, data = image.uri.split(',', 1) + img_data = base64.b64decode(data) + img = Image.open(io.BytesIO(img_data)) + else: + # File path + img_path = os.path.join(base_dir, image.uri) + img = Image.open(img_path) + else: + # Embedded in bufferView + buffer_view = self.gltf.bufferViews[image.bufferView] + data = self.gltf.binary_blob() + offset = buffer_view.byteOffset + length = buffer_view.byteLength + img_data = data[offset:offset+length] + img = Image.open(io.BytesIO(img_data)) + + # Convert to RGBA if needed + if img.mode != 'RGBA': + img = img.convert('RGBA') + + self.model.textures.append(img) + print(f"Loaded texture {tex_idx}: {img.size}") + + except Exception as e: + print(f"Failed to load texture {tex_idx}: {e}") + self.model.textures.append(None) diff --git a/src/ui/vrm_widget.py b/src/ui/vrm_widget.py index d13c075..9c7d861 100644 --- a/src/ui/vrm_widget.py +++ b/src/ui/vrm_widget.py @@ -1,6 +1,8 @@ """ VRM Widget - OpenGL widget for rendering VRM models """ +import os +from typing import Optional from PyQt6.QtOpenGLWidgets import QOpenGLWidget from PyQt6.QtCore import Qt, QTimer from PyQt6.QtGui import QSurfaceFormat @@ -9,6 +11,7 @@ from OpenGL.GLU import * import numpy as np from src.core.state_manager import StateManager, EmotionState +from src.rendering.vrm_loader import VRMLoader, VRMModel class VRMWidget(QOpenGLWidget): """OpenGL widget for rendering VRM character model""" @@ -22,9 +25,13 @@ class VRMWidget(QOpenGLWidget): super().__init__(parent) self.state_manager = state_manager - self.vrm_model = None + self.vrm_model: Optional[VRMModel] = None self.current_emotion = EmotionState.NEUTRAL + # OpenGL resources + self.textures = [] + self.display_lists = [] + # Setup state change listener self.state_manager.register_listener('emotion_change', self.on_emotion_change) @@ -38,18 +45,29 @@ class VRMWidget(QOpenGLWidget): # Enable depth testing glEnable(GL_DEPTH_TEST) + # Disable backface culling (VRM models sometimes have inverted faces) + glDisable(GL_CULL_FACE) + # Enable blending for transparency glEnable(GL_BLEND) glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) + # Enable lighting for basic shading + glEnable(GL_LIGHTING) + glEnable(GL_LIGHT0) + glEnable(GL_COLOR_MATERIAL) + glColorMaterial(GL_FRONT_AND_BACK, GL_AMBIENT_AND_DIFFUSE) + + # Set up basic light (brighter for better visibility) + glLightfv(GL_LIGHT0, GL_POSITION, [0.0, 1.0, 2.0, 0.0]) + glLightfv(GL_LIGHT0, GL_AMBIENT, [0.5, 0.5, 0.5, 1.0]) + glLightfv(GL_LIGHT0, GL_DIFFUSE, [1.0, 1.0, 1.0, 1.0]) + # Set clear color (transparent) glClearColor(0.0, 0.0, 0.0, 0.0) - # TODO: Load VRM model - # self.load_vrm_model() - - # TODO: Setup shaders (MToon shader) - # self.setup_shaders() + # Load VRM model + self._load_default_model() print("OpenGL initialized") @@ -73,12 +91,18 @@ class VRMWidget(QOpenGLWidget): # Reset modelview matrix glLoadIdentity() - # Move camera back - glTranslatef(0.0, -0.5, -3.0) + # Move camera back and adjust vertical position to show full body + glTranslatef(0.0, -0.5, -2.2) - # TODO: Render VRM model - # For now, render a placeholder - self.render_placeholder() + # Render VRM model if loaded, otherwise show placeholder + if self.vrm_model: + # Rotate model to face camera (VRM models typically face +Z, we need -Z) + glRotatef(180, 0, 1, 0) + # Scale down VRM models (they're usually in meters, about 1.6 units tall) + glScalef(0.85, 0.85, 0.85) + self.render_vrm() + else: + self.render_placeholder() def render_placeholder(self): """Render a placeholder (cube) until VRM is loaded""" @@ -144,8 +168,135 @@ class VRMWidget(QOpenGLWidget): # TODO: Update blend shapes/expressions based on emotion print(f"VRM Widget: Emotion changed to {new_emotion.value}") + def _load_default_model(self): + """Load the default VRM model from models folder""" + models_dir = os.path.join(os.getcwd(), 'models') + + # Look for .vrm files in models directory + if os.path.exists(models_dir): + vrm_files = [f for f in os.listdir(models_dir) if f.endswith('.vrm')] + if vrm_files: + model_path = os.path.join(models_dir, vrm_files[0]) + self.load_vrm_model(model_path) + else: + print("No VRM model found in models/ folder") + else: + print("models/ directory not found") + def load_vrm_model(self, model_path: str): """Load VRM model from file""" - # TODO: Implement VRM loading using pygltflib - print(f"Loading VRM model from {model_path}") - pass + try: + print(f"Loading VRM model from {model_path}") + loader = VRMLoader() + self.vrm_model = loader.load(model_path) + + # Create OpenGL textures + self._create_textures() + + print(f"VRM model loaded successfully: {len(self.vrm_model.meshes)} meshes") + + except Exception as e: + print(f"Failed to load VRM model: {e}") + import traceback + traceback.print_exc() + + def _create_textures(self): + """Create OpenGL textures from loaded images""" + if not self.vrm_model or not self.vrm_model.textures: + print("No textures to create") + return + + print(f"Creating {len(self.vrm_model.textures)} OpenGL textures...") + for idx, img in enumerate(self.vrm_model.textures): + if img is None: + print(f" Texture {idx}: None (skipped)") + self.textures.append(None) + continue + + tex_id = glGenTextures(1) + glBindTexture(GL_TEXTURE_2D, tex_id) + + # Convert PIL image to bytes + img_data = img.tobytes() + width, height = img.size + + # Upload texture + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, img_data) + + # Set texture parameters + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT) + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT) + + self.textures.append(tex_id) + print(f" Texture {idx}: {width}x{height}, ID={tex_id}") + + def render_vrm(self): + """Render the loaded VRM model""" + if not self.vrm_model: + return + + # Enable texturing + glEnable(GL_TEXTURE_2D) + + # Set texture environment to modulate (combine texture with lighting) + glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE) + + # Render each mesh + for mesh in self.vrm_model.meshes: + for primitive in mesh['primitives']: + self._render_primitive(primitive) + + glDisable(GL_TEXTURE_2D) + + def _render_primitive(self, primitive): + """Render a single primitive (submesh)""" + vertices = primitive.get('vertices') + normals = primitive.get('normals') + uvs = primitive.get('uvs') + indices = primitive.get('indices') + material_idx = primitive.get('material_idx', 0) + + if vertices is None: + return + + # Get material + has_texture = False + if material_idx < len(self.vrm_model.materials): + material = self.vrm_model.materials[material_idx] + + # Bind texture if available + tex_idx = material.get('texture_idx') + if tex_idx is not None and tex_idx < len(self.textures) and self.textures[tex_idx]: + glBindTexture(GL_TEXTURE_2D, self.textures[tex_idx]) + has_texture = True + # Use white color when textured so texture shows through properly + glColor4f(1.0, 1.0, 1.0, 1.0) + else: + glBindTexture(GL_TEXTURE_2D, 0) + # Use material base color when no texture + color = material.get('base_color', [1.0, 1.0, 1.0, 1.0]) + glColor4fv(color) + + # Render with immediate mode + if indices is not None: + glBegin(GL_TRIANGLES) + for idx in indices: + if normals is not None and idx < len(normals): + glNormal3fv(normals[idx]) + if uvs is not None and idx < len(uvs): + glTexCoord2fv(uvs[idx]) + if idx < len(vertices): + glVertex3fv(vertices[idx]) + glEnd() + else: + # No indices, render directly + glBegin(GL_TRIANGLES) + for i in range(len(vertices)): + if normals is not None and i < len(normals): + glNormal3fv(normals[i]) + if uvs is not None and i < len(uvs): + glTexCoord2fv(uvs[i]) + glVertex3fv(vertices[i]) + glEnd()