diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index 0bb8c624..0fdf74ee 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -1,31 +1,33 @@ -name: Build Documentation using MkDocs -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch -# https://medium.com/dev-genius/continuous-deployment-for-mkdocs-with-github-actions-7dceec87f0ea -on: - push: - branches: [master,main] - pull_request: - branches: [master,main] -jobs: - build: - name: Build and Deploy Documentation - runs-on: ubuntu-latest - steps: - - name: Checkout Master - uses: actions/checkout@v2 +name: Build Documentation using MkDocs +# Builds the docs (strict, so broken links fail the check) and deploys them to the gh-pages branch, +# on pushes to main and on pull requests. +on: + push: + branches: [main] + paths: ['Documentation/**', '.github/workflows/mkdocs.yml'] + pull_request: + branches: [main] + paths: ['Documentation/**', '.github/workflows/mkdocs.yml'] +permissions: + contents: write +jobs: + build: + name: Build and Deploy Documentation + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 # full history for the git-revision-date plugin - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install mkdocs-material - pip install mkdocs-git-revision-date-localized-plugin - pip install mkdocs-git-committers-plugin-2 - pip install mkdocs-git-authors-plugin - pip install mkdocs-glightbox - - name: Deploy - run: | - mkdocs gh-deploy --config-file Documentation/mkdocs.yml --remote-branch gh-pages --force + uses: actions/setup-python@v7 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install mkdocs-material mkdocs-git-revision-date-localized-plugin mkdocs-glightbox + - name: Build + run: mkdocs build --strict --config-file Documentation/mkdocs.yml + - name: Deploy + run: mkdocs gh-deploy --config-file Documentation/mkdocs.yml --remote-branch gh-pages --force diff --git a/.gitignore b/.gitignore index 915908fa..5ded57db 100644 --- a/.gitignore +++ b/.gitignore @@ -73,9 +73,7 @@ crashlytics-build.properties # VSCode folder .vscode/ -# Ignore auto-generated XR because these are per-end user settings -/Assets/XR/ -/Assets/XR.meta +# Assets/XR (XR loader and OpenXR settings) is tracked on purpose so everyone builds with the same settings Recordings/ diff --git a/Assets/Lab/Core/Runtime/Art/Icons/Name.png.meta b/Assets/Lab/Core/Runtime/Art/Icons/Name.png.meta index 0e96e1d7..0686991b 100644 --- a/Assets/Lab/Core/Runtime/Art/Icons/Name.png.meta +++ b/Assets/Lab/Core/Runtime/Art/Icons/Name.png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: 303d29210c9b39446b1f6bc0e9d029be TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: 7405661294618894375 + second: Name_0 externalObjects: {} - serializedVersion: 11 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,10 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -62,8 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 + cookieLightType: 1 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -73,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -85,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Android maxTextureSize: 2048 resizeAlgorithm: 0 @@ -97,12 +106,36 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: Name_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 1024 + height: 1024 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: 7206e98e5b136c660800000000000000 + internalID: 7405661294618894375 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -112,9 +145,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] - spritePackingTag: + spriteCustomMetadata: + entries: [] + nameFileIdTable: {} + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Lab/Core/Runtime/Art/Icons/volume-play-white.png.meta b/Assets/Lab/Core/Runtime/Art/Icons/volume-play-white.png.meta index 9405cbb5..26e1a2ea 100644 --- a/Assets/Lab/Core/Runtime/Art/Icons/volume-play-white.png.meta +++ b/Assets/Lab/Core/Runtime/Art/Icons/volume-play-white.png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: 7b53e9eb18bf45341b03461b55a192b5 TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: 1061623916413588505 + second: volume-play-white_0 externalObjects: {} - serializedVersion: 11 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,10 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -62,8 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 + cookieLightType: 1 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -73,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -85,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Android maxTextureSize: 2048 resizeAlgorithm: 0 @@ -97,12 +106,36 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: volume-play-white_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 640 + height: 512 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: 91414e6c355abbe00800000000000000 + internalID: 1061623916413588505 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -112,9 +145,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] - spritePackingTag: + spriteCustomMetadata: + entries: [] + nameFileIdTable: {} + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/edit_FILL1_wght700_GRAD0_opsz48.png.meta b/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/edit_FILL1_wght700_GRAD0_opsz48.png.meta index 2a775a8a..63347e20 100644 --- a/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/edit_FILL1_wght700_GRAD0_opsz48.png.meta +++ b/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/edit_FILL1_wght700_GRAD0_opsz48.png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: d233156b35457ee4c9ed3c1807d5fd91 TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: -7877580773724871505 + second: edit_FILL1_wght700_GRAD0_opsz48_0 externalObjects: {} - serializedVersion: 12 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,11 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 - ignoreMasterTextureLimit: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -63,9 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 cookieLightType: 0 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -75,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -87,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Server maxTextureSize: 2048 resizeAlgorithm: 0 @@ -99,9 +106,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Android maxTextureSize: 2048 resizeAlgorithm: 0 @@ -111,12 +119,36 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: edit_FILL1_wght700_GRAD0_opsz48_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 48 + height: 48 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: fa4aea406063da290800000000000000 + internalID: -7877580773724871505 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -126,10 +158,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] + spriteCustomMetadata: + entries: [] nameFileIdTable: {} - spritePackingTag: + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/horizontal_rule_FILL1_wght700_GRAD0_opsz48.png.meta b/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/horizontal_rule_FILL1_wght700_GRAD0_opsz48.png.meta index 43b05ced..3fd55167 100644 --- a/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/horizontal_rule_FILL1_wght700_GRAD0_opsz48.png.meta +++ b/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/horizontal_rule_FILL1_wght700_GRAD0_opsz48.png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: cddf3f5a5bd7b4a46af42f38b466244c TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: 5920816587221518968 + second: horizontal_rule_FILL1_wght700_GRAD0_opsz48_0 externalObjects: {} - serializedVersion: 12 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,11 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 - ignoreMasterTextureLimit: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -63,9 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 cookieLightType: 0 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -75,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -87,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Server maxTextureSize: 2048 resizeAlgorithm: 0 @@ -99,9 +106,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Android maxTextureSize: 2048 resizeAlgorithm: 0 @@ -111,12 +119,36 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: horizontal_rule_FILL1_wght700_GRAD0_opsz48_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 48 + height: 48 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: 87a4495f057fa2250800000000000000 + internalID: 5920816587221518968 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -126,10 +158,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] + spriteCustomMetadata: + entries: [] nameFileIdTable: {} - spritePackingTag: + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/save_FILL1_wght700_GRAD0_opsz48.png.meta b/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/save_FILL1_wght700_GRAD0_opsz48.png.meta index ab96c55d..c45f3869 100644 --- a/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/save_FILL1_wght700_GRAD0_opsz48.png.meta +++ b/Assets/Lab/Core/Runtime/Resources/VAInteractableObjects/save_FILL1_wght700_GRAD0_opsz48.png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: 7384d48146335344aa51c431428f3106 TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: -7185052849241797039 + second: save_FILL1_wght700_GRAD0_opsz48_0 externalObjects: {} - serializedVersion: 12 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,11 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 - ignoreMasterTextureLimit: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -63,9 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 cookieLightType: 0 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -75,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -87,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Server maxTextureSize: 2048 resizeAlgorithm: 0 @@ -99,9 +106,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Android maxTextureSize: 2048 resizeAlgorithm: 0 @@ -111,12 +119,36 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: save_FILL1_wght700_GRAD0_opsz48_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 48 + height: 48 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: 15ec88c6080994c90800000000000000 + internalID: -7185052849241797039 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -126,10 +158,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] + spriteCustomMetadata: + entries: [] nameFileIdTable: {} - spritePackingTag: + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_Germany.png.meta b/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_Germany.png.meta index 59fb2fed..b3558686 100644 --- a/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_Germany.png.meta +++ b/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_Germany.png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: 681c25958809b44bd8daaceece3dcd09 TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: 1572207362141225981 + second: Flag_of_Germany_0 externalObjects: {} - serializedVersion: 12 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,11 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 - ignoreMasterTextureLimit: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -63,9 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 cookieLightType: 0 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -75,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -87,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Server maxTextureSize: 2048 resizeAlgorithm: 0 @@ -99,12 +106,49 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + - serializedVersion: 4 + buildTarget: Android + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: Flag_of_Germany_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 1000 + height: 600 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: df38ea5144a91d510800000000000000 + internalID: 1572207362141225981 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -114,10 +158,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] + spriteCustomMetadata: + entries: [] nameFileIdTable: {} - spritePackingTag: + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_the_United_Kingdom_(1-2).png.meta b/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_the_United_Kingdom_(1-2).png.meta index 32ae22a4..401c1d47 100644 --- a/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_the_United_Kingdom_(1-2).png.meta +++ b/Assets/Lab/Core/Samples/Objects/Materials/Language/Flag_of_the_United_Kingdom_(1-2).png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: b727bfffc36524403a4cc0f75d4831b7 TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: 4047080015453609154 + second: Flag_of_the_United_Kingdom_(1-2)_0 externalObjects: {} - serializedVersion: 12 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,11 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 - ignoreMasterTextureLimit: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -63,9 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 cookieLightType: 0 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -75,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -87,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Server maxTextureSize: 2048 resizeAlgorithm: 0 @@ -99,12 +106,49 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + - serializedVersion: 4 + buildTarget: Android + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: Flag_of_the_United_Kingdom_(1-2)_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 1200 + height: 600 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: 2ccd1bae4dd1a2830800000000000000 + internalID: 4047080015453609154 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -114,10 +158,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] + spriteCustomMetadata: + entries: [] nameFileIdTable: {} - spritePackingTag: + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Lab/Core/Samples/Objects/Materials/Language/globe-solid 1.png.meta b/Assets/Lab/Core/Samples/Objects/Materials/Language/globe-solid 1.png.meta index 3f2a27c8..16f4b5bd 100644 --- a/Assets/Lab/Core/Samples/Objects/Materials/Language/globe-solid 1.png.meta +++ b/Assets/Lab/Core/Samples/Objects/Materials/Language/globe-solid 1.png.meta @@ -1,9 +1,12 @@ fileFormatVersion: 2 guid: dc63444fc13444be6a8f8dd3c6b52d14 TextureImporter: - internalIDToNameTable: [] + internalIDToNameTable: + - first: + 213: -5420819394531046391 + second: globe-solid 1_0 externalObjects: {} - serializedVersion: 12 + serializedVersion: 13 mipmaps: mipMapMode: 0 enableMipMap: 0 @@ -20,11 +23,12 @@ TextureImporter: externalNormalMap: 0 heightScale: 0.25 normalMapFilter: 0 + flipGreenChannel: 0 isReadable: 0 streamingMipmaps: 0 streamingMipmapsPriority: 0 vTOnly: 0 - ignoreMasterTextureLimit: 0 + ignoreMipmapLimit: 0 grayScaleToAlpha: 0 generateCubemap: 6 cubemapConvolution: 0 @@ -63,9 +67,10 @@ TextureImporter: textureFormatSet: 0 ignorePngGamma: 0 applyGammaDecoding: 0 + swizzle: 50462976 cookieLightType: 0 platformSettings: - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: DefaultTexturePlatform maxTextureSize: 2048 resizeAlgorithm: 0 @@ -75,9 +80,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Standalone maxTextureSize: 2048 resizeAlgorithm: 0 @@ -87,9 +93,10 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 - - serializedVersion: 3 + - serializedVersion: 4 buildTarget: Server maxTextureSize: 2048 resizeAlgorithm: 0 @@ -99,12 +106,49 @@ TextureImporter: crunchedCompression: 0 allowsAlphaSplitting: 0 overridden: 0 + ignorePlatformSupport: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + - serializedVersion: 4 + buildTarget: Android + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + ignorePlatformSupport: 0 androidETC2FallbackOverride: 0 forceMaximumCompressionQuality_BC6H_BC7: 0 spriteSheet: serializedVersion: 2 - sprites: [] + sprites: + - serializedVersion: 2 + name: globe-solid 1_0 + rect: + serializedVersion: 2 + x: 0 + y: 0 + width: 512 + height: 512 + alignment: 0 + pivot: {x: 0, y: 0} + border: {x: 0, y: 0, z: 0, w: 0} + customData: + outline: [] + physicsShape: [] + tessellationDetail: -1 + bones: [] + spriteID: 90ccd6d3b7165c4b0800000000000000 + internalID: -5420819394531046391 + vertices: [] + indices: + edges: [] + weights: [] outline: [] + customData: physicsShape: [] bones: [] spriteID: 5e97eb03825dee720800000000000000 @@ -114,10 +158,11 @@ TextureImporter: edges: [] weights: [] secondaryTextures: [] + spriteCustomMetadata: + entries: [] nameFileIdTable: {} - spritePackingTag: + mipmapLimitGroupName: pSDRemoveMatte: 0 - pSDShowRemoveMatteOption: 0 userData: assetBundleName: assetBundleVariant: diff --git a/Assets/Oculus.meta b/Assets/Oculus.meta new file mode 100644 index 00000000..df01e2ef --- /dev/null +++ b/Assets/Oculus.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 7978697dc4b45c2499315e9b6fc5b36e +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Samples.meta b/Assets/Samples.meta new file mode 100644 index 00000000..39968bc4 --- /dev/null +++ b/Assets/Samples.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: eb2502cdffa176844948b16ccb48c8f3 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Documentation/docs/getting_started/adding_scenes.md b/Documentation/docs/getting_started/adding_scenes.md index e210adde..c64a1b66 100644 --- a/Documentation/docs/getting_started/adding_scenes.md +++ b/Documentation/docs/getting_started/adding_scenes.md @@ -4,13 +4,17 @@ you must first add the scene to the Unity project and then to the MongoDB databa ## Add Scene in Unity ### Create a new Scene -- Scenes are placed in the `Assets/Local/Scenes` folder. -- Right-click in the `Assets/Local/Scenes` folder and select `Create > Scene`. -- Add and activate scene in the build settings `File/Build Setting > Add Open Scene`. +- Place your scenes in a folder of your own project, e.g. `Assets/Scenes`, not inside `Assets/Lab`. +- The easiest start is a copy of the `Example` sample scene (see [Special Scenes](special_scenes.md)). It already + contains the `Social Network Scene` prefab for networking. +- Don't add a player or camera: the [player](player_controller.md) comes from MetaStart. +- Add a GameObject for each spawn point and name it like the role's `spawnPosition` (e.g. `DefaultSpawn`). +- Add the scene to the build profile (*File > Build Profiles > Scene List*), after MetaStart. #### Debug Scene without MetaStart -Add the following prefabs: `Social Network Scene`, `MetaPlayer`, `AvatarSdkManagerHorizon` to the scene. -(Not testet. `Social Network Scene` e.g. is not nessessary for the scene to work.) +To test a scene on its own, temporarily add the prefabs `MetaPlayer2.0` and `AvatarSdkManagerStyle2Meta` (from the +Meta Avatars *Sample Scenes*, folder `Common/Prefabs/AvatarSdkManager/Recommended`), as MetaStart does. Rename the +manager to `AvatarSdkManagerMeta`: the avatar scripts look it up by that name. Remove both again before building. ## Add Scene to the Database The Scene must be added to the database to be accessible via MetaStart. There are @@ -22,171 +26,72 @@ two ways to do this: don't need direct database access - just an API key. Start there; the walkthrough builds exactly the structure documented below, one call at a time. - **Directly in MongoDB.** As a student, ask your corresponding supervisor to add - the scene for you. The scenes live in `Experiment/scenarios-languages`, in the + the scene for you. Scenes are stored in the `scenarios` collection, in the structure below. -The .json structure - what either route produces - is as follows: +This is the structure of a scene, as `GET /scene` returns it and as the client reads it (`ApiScene` in +`Assets/Lab/Core/Runtime/Scripts/SceneManagement/SceneManagerTypes.cs`): -``` json title="Parameter Explanation" -{ - "name": Name of the Scene in the scene selector, - "internalName": Name of the Scene in Unity, - "roles": Roles in the Scenario. Currently supporting english and german - { - "name": Name of the Role, - "description": Description of the Role, - "spawnPosition": Spawn Position of the Role, - "mode": Mode of the Role (player, observer), - "maxCount": Maximum Count of the Role, - "admin": Admin Role (start and ending levels), - "level": Level Descriptions for that role - }, - "level": Levels in the Scenario, - { - "id": ID of the Level, - "delay": Delay of the Level (stops the level after n minutes; null if no delay) - }, - "enable": Visible via MetaStart, - "amountPlayersRequired": Amount of Players Required, until admin can start the scenario -} -``` +| Field | Meaning | +|---|---| +| `name`, `shortName` | Name in the scene selector, and a short version where space is limited | +| `author` | Author of the scenario | +| `internalName` | Name of the Unity scene | +| `enabled` | Visible in MetaStart | +| `roles` | Roles by identifier (your own key, e.g. `interviewee`) | +| `roles..mode` | `player` or `spectator` (`webcam` exists but is not used by the client yet) | +| `roles..spawnPosition` | Name of the GameObject in the scene the role spawns at | +| `roles..maxCount` | Maximum number of players with this role | +| `roles..admin` | The admin starts and ends the levels | +| `roles..restrictions` | Restrictions of the role, e.g. `blur` | +| `roles..locales.` | `name` and `description` of the role per language (`EN`, `DE`) | +| `levels` | The stages of the scenario, in order | +| `levels[].delay` | Stops the level after n minutes (`null`: no limit) | +| `levels[].roleDescriptions..locales.` | What the role does in this level | + +The example below defines the `InterView` sample scene: an interviewee in VR who starts and ends the level, and an +optional spectator. The spawn points `DefaultSpawn` and `spectator` exist in that scene. ``` json title="Example Scene Entry" { - "name": "Praktikum Experiment 3", - "shortName": "Praktikum 3", - "author": "Alexander Henlein", - "internalName": "PraktikumScenario03", + "name": "InterView Sample", + "shortName": "InterView", + "author": "Jane Doe", + "internalName": "InterView", + "enabled": true, "roles": { - "DE": [ - { - "name": "Person 1", - "description": [ - { - "id": 1, - "description": "Rundfahrt erklären" - } - ], - "spawnPosition": "P1", - "mode": "player", - "maxCount": 1, - "admin": true, - "level": [ - { - "id": 1, - "description": "Rundfahrt" - }, - { - "id": 2, - "description": "Erklären" - }, - { - "id": 3, - "description": "Warten" - } - ] - }, - { - "name": "Person 2", - "description": [ - { - "id": 1, - "description": "Weg finden" - } - ], - "spawnPosition": "P2", - "mode": "player", - "maxCount": 1, - "admin": false, - "level": [ - { - "id": 1, - "description": "Warten" - }, - { - "id": 2, - "description": "Erklären lassen" - }, - { - "id": 3, - "description": "Finden" - } - ] + "interviewee": { + "mode": "player", + "spawnPosition": "DefaultSpawn", + "maxCount": 1, + "admin": true, + "restrictions": [], + "locales": { + "EN": { "name": "Interviewee", "description": ["Answer the interviewer's questions."] }, + "DE": { "name": "Befragte Person", "description": ["Beantworte die Fragen der interviewenden Person."] } } - ], - "EN": [ - { - "name": "Person 1", - "description": [ - { - "id": 1, - "description": "Rundfahrt" - } - ], - "spawnPosition": "P1", - "mode": "player", - "maxCount": 1, - "admin": true, - "level": [ - { - "id": 1, - "description": "Rundfahrt" - }, - { - "id": 2, - "description": "Erklären" - }, - { - "id": 3, - "description": "Warten" - } - ] - }, - { - "name": "Person 2", - "description": [ - { - "id": 1, - "description": "Erklären lassen" - } - ], - "spawnPosition": "P2", - "mode": "player", - "maxCount": 1, - "admin": false, - "level": [ - { - "id": 1, - "description": "Warten" - }, - { - "id": 2, - "description": "Erklären lassen" - }, - { - "id": 3, - "description": "Finden" - } - ] + }, + "spectator": { + "mode": "spectator", + "spawnPosition": "spectator", + "maxCount": 1, + "admin": false, + "restrictions": [], + "locales": { + "EN": { "name": "Spectator", "description": ["Watch the interview."] }, + "DE": { "name": "Beobachtung", "description": ["Beobachte das Interview."] } } - ] + } }, - "level": [ + "levels": [ { "id": 1, - "delay": null - }, - { - "id": 2, - "delay": null - }, - { - "id": 3, - "delay": null + "delay": null, + "roleDescriptions": { + "interviewee": { "locales": { "EN": { "description": ["Interview"] }, "DE": { "description": ["Interview"] } } }, + "spectator": { "locales": { "EN": { "description": ["Observe"] }, "DE": { "description": ["Beobachten"] } } } + } } - ], - "enable": true, - "amountPlayersRequired": 2 + ] } ``` - diff --git a/Documentation/docs/getting_started/interview_client.md b/Documentation/docs/getting_started/interview_client.md index 7e7702ac..e2e08327 100644 --- a/Documentation/docs/getting_started/interview_client.md +++ b/Documentation/docs/getting_started/interview_client.md @@ -23,31 +23,34 @@ itself, its services and their deployment are documented separately: ## What the client connects to -Four independent services. Each is configured in a different place, and each can -fail without the others noticing. +Four independent services, all configured in the [endpoints file](../modules/endpoints.md) +(`Assets/Resources/VasiliEndpoints.asset`). Each can fail without the others noticing. -| Service | Purpose | Configured in | +| Service | Purpose | Endpoints field | |---|---|---| -| Janus | Audio/video and the control data channel | `WebCamClient` | -| Ubiq | VR room and avatar state | `RoomClient` connection definition | -| Logging API | Tracking data and events | Va.Si.Li API asset | -| Questionnaire | The survey instrument | `InterviewTokenManager` | +| Janus | Audio/video and the control data channel | `janus`, `iceServers` | +| Ubiq | VR room and avatar state | `ubiqServer` | +| Logging API | Tracking data and events | `api` | +| Questionnaire | The survey instrument | `questionnaire` | ## Configuration -Open the FACES scene and configure the components below. +The InterView sample scenes are in `Assets/Lab/Core/Samples/Scenes`: `InterViewStart` (token entry, holds the +`InterviewTokenManager`), `InterView` (the interview room, holds the `WebCamClient`) and `InterViewReturn`. Add all +three to the build profile, `InterViewStart` first, and check the components below. ### `WebCamClient` The Janus client. Publishes local audio and video, subscribes to the interviewer's feed, and hosts the textroom data channel that carries chat, -answer options, gaze updates and the recording signal. +answer options, gaze updates and the recording signal. Its server comes from +`janus` in the endpoints: use `wss://janus.`, the TLS endpoint of your +reverse proxy, not Janus's own port. `iceServers` must match the TURN server the +backend uses. | Field | Value | Notes | |---|---|---| -| `ServerUrl` | `wss://janus.` | Must be `wss://` - the TLS endpoint of your reverse proxy, not Janus's own port | | `SubProtocol` | `janus-protocol` | Leave unchanged | -| `IceServers` | TURN URL, username, credential | Must match the TURN server the backend uses | | `RecordingPath` | `/mnt/recordings/` | Path **inside the Janus container**; the room ID is appended automatically | | `captureWidth` / `captureHeight` | `854` × `480` | | | `maxBitrate` / `minBitrate` | `1000000` / `250000` | | @@ -59,14 +62,13 @@ Holds the participant token across scene loads and derives every identifier from it, so the headset and the interviewer's browser meet without a coordination step. -| Field | Value | -|---|---| -| `BrowserBaseUrl` | Your questionnaire URL. The token is appended as `?token=…&room=…` | +The questionnaire URL is `questionnaire` in the endpoints. The token is appended as `?token=…&room=…`. The follow-up +questionnaire that the interviewer can send afterwards is `followupQuestionnaire`. ``` token ──┬─→ RoomId (Janus room ID) ├─→ UbiqRoomName = "interview-{token}" - └─→ BrowserUrl = {BrowserBaseUrl}?token={token}&room={token} + └─→ BrowserUrl = {questionnaire}?token={token}&room={token} ``` The room ID is the token parsed as an integer when it is all digits, and @@ -80,15 +82,14 @@ otherwise a DJB2 hash (seed 5381, multiplier 33), absolute value, minimum 1. ### Ubiq connection -Set the `RoomClient` connection definition to your Ubiq host and its published -TCP port (**16485** in the default deployment). +Set `ubiqServer` in the endpoints to a Ubiq connection definition with your Ubiq host +and its published TCP port (**16485** in the default deployment). ### Logging API -Set the Va.Si.Li API asset `host` to your API URL, and the API key to the same -value the backend was deployed with. +Set `api` in the endpoints to your Database API URL. -!!! warning "A wrong API key loses the whole session, silently" +!!! warning "A wrong API address loses the whole session, silently" Logging failures do not interrupt an interview. The session looks completely normal and records nothing. Verify that data is arriving **before** a participant sits down, not afterwards. @@ -101,11 +102,11 @@ value the backend was deployed with. ## Pre-session checklist -- [ ] `ServerUrl` reachable from the headset's network +- [ ] `janus` reachable from the headset's network - [ ] TURN credentials current and matching the server - [ ] Ubiq host and port correct -- [ ] API key matches the backend -- [ ] `BrowserBaseUrl` points at the correct questionnaire +- [ ] `api` points at the running Database API +- [ ] `questionnaire` points at the correct questionnaire - [ ] A full test interview completed, with data confirmed in the database ## Troubleshooting diff --git a/Documentation/docs/getting_started/meta_avatar.md b/Documentation/docs/getting_started/meta_avatar.md index 1cf101d7..db373081 100644 --- a/Documentation/docs/getting_started/meta_avatar.md +++ b/Documentation/docs/getting_started/meta_avatar.md @@ -1,6 +1,6 @@ # Meta Avatar -Instead of the biq Avatars, we are using the [Meta Avatar](https://developer.oculus.com/documentation/unity/meta-avatars-overview/) system. +Instead of the Ubiq avatars, we are using the [Meta Avatar](https://developers.meta.com/horizon/documentation/unity/meta-avatars-overview/) system. The avatars have the following advantages: * Diverse cast of pre-built avatars + customization options @@ -9,10 +9,13 @@ The avatars have the following advantages: * Visualisation of Facial Expressions ## Usage -At the moment the meta avatars are still loaded via an extra private Git project. -In the next version, these will be available as a standard module of the main project. +The Meta Avatars SDK (`com.meta.xr.sdk.avatars`, 40.0.1) is installed from the Unity registry as a dependency of +Va.Si.Li-Lab. Its *Sample Scenes* sample must be imported, because Va.Si.Li-Lab uses `SampleAvatarEntity` and the +`AvatarSdkManagerStyle2Meta` prefab from it (in MetaStart, renamed to `AvatarSdkManagerMeta`, the name the avatar scripts look up), see [Setting Up](setting_up.md#import-the-required-samples). + +The avatar scripts of Va.Si.Li-Lab are in `Assets/Lab/Core/Runtime/Scripts/MetaAvatars`. The local avatar is driven by +`MetaAvatarInput` on the [player](player_controller.md). ## References -* [Meta Avatar Documentation](https://developer.oculus.com/documentation/unity/meta-avatars-overview/) -* [Meta Avatar SDK](https://assetstore.unity.com/packages/tools/integration/meta-avatars-sdk-271958) -* [Meta Avatar SDK Sample](https://assetstore.unity.com/packages/tools/integration/meta-avatars-sdk-sample-assets-272863) \ No newline at end of file +* [Meta Avatar Documentation](https://developers.meta.com/horizon/documentation/unity/meta-avatars-overview/) +* [Meta Avatars SDK on the Unity registry](https://docs.unity3d.com/Manual/upm-ui.html) (*Package Manager > Unity Registry > Meta Avatars SDK*) diff --git a/Documentation/docs/getting_started/meta_interactions.md b/Documentation/docs/getting_started/meta_interactions.md deleted file mode 100644 index 8db94bfc..00000000 --- a/Documentation/docs/getting_started/meta_interactions.md +++ /dev/null @@ -1,68 +0,0 @@ -## OVRCameraRigInteraction - -If you have the CoreSDK/InteractionSDK Assets installed, delete the Default Camera in your scene and just add this prefab into your scene. This Camera works both with Controllers and Handtracking by default. - - - -## Usage -The (hand) interactions made possible with this setup include: - -* Grabs (Special Grabs include i.e. Pinch and Pointer) -* Distance Grabs -* Ray Grabs -* Locomotion -* Teleportation - -## How to Add Interactions -To add interactions to your OVRCameraRig, all that is required is to modify the state of the object you want to make interactable. - -For each type of grab interaction, follow these steps: - -Highlight your GameObject, right-click, search for INTERACTION SDK, and choose the type of grab interaction you want to apply to your GameObject. -If your object doesn’t have a Collider or Rigidbody yet, a window will pop up prompting you to click "Create." This will add a child GameObject named "ISDK_...", allowing the object to be interacted with. (This only works because the OVRCameraRigInteraction already has all of the necessary interactors.) -It is recommended NOT to attach every type of interactor to a single GameObject. If you have a large object but only want a small part of it to be interactable, the same rule applies. - -There are multiple types of interactions that you can add to a GameObject: -* Hand Grab -* Distance Grab -* Poke -* Ray Grab -* etc. - -[For more check out this link](https://developers.meta.com/horizon/documentation/unity/unity-isdk-interaction-sdk-overview/) - -## Custom Interactions - -To add custom interactions for hand/controllers, you will need to reference the OVRHand class or the appropriate controller class for the controllers you’re using. You can also directly read button inputs using the OVRInput class. For an example of both methods, refer to the MetaPen.cs script, or check the official documentation for [more information](https://developers.meta.com/horizon/documentation/unity/unity-inputactions). - -## Locomotion - -The OVRCameraRigInteractable prefab already comes with the necessary setup to implement teleporting and turning via hand tracking. -To enable teleportation, you will need to copy HotspotVoidFloor and NavMeshHotspotWalkable from the Meta Locomotion sample scene. This will create a large plane that allows for teleportation. If you want to prevent teleportation through objects, you will also need to copy CollidersTeleportBlocker. - -It may be necessary to adjust the Y-axis of HotspotVoidFloor to ensure the floor does not block teleportation. - -Alternatively, for more complex environments, you can use the "Add Teleport Quick Action" from the Oculus SDK. You can also bake a NavMesh and add a teleport interactable to it. However, this is not recommended. - -## Scirpts -### FireworkSpawn.cs -Added the Firework Spawn script that takes Spawn Object, Spawn Point, Height Offset, Upward Force. -It spawns an object with upward force (firework) when interacted with via a RayInteractor (Hand and Controller). - - -### Metamovement.cs -Component script in OVRCameraRig to make sure that the left Oculus-joystick of the Player ensures forward/backward/left/right movement even if head is turned. - -### TeleportOnTrigger.cs -Added logic from previous, changed the way it teleports a player on entry. Updated teleport spots to stop teleportation from occuring in the ground. - - -### PraktikumScenario01_RoleManager.cs -Changed varaibles to fit new camera setup. Tweaked logic to ensure it handles the player correctly. - - - - -### References -[MetaInteractionSDK](https://developer.oculus.com/documentation/unity/unity-isdk-interaction-sdk-overview/) -[HandTracking](https://developer.oculus.com/documentation/unity/unity-handtracking-overview/) diff --git a/Documentation/docs/getting_started/player_controller.md b/Documentation/docs/getting_started/player_controller.md new file mode 100644 index 00000000..75f04b48 --- /dev/null +++ b/Documentation/docs/getting_started/player_controller.md @@ -0,0 +1,58 @@ +# Player Controller + +The player is the prefab `MetaPlayer2.0` in `Assets/Lab/Core/Runtime/Prefabs`. It is built from Meta XR +[Building Blocks](https://developers.meta.com/horizon/documentation/unity/bb-overview/) and works with controllers and +hand tracking. + +## How it is used +- Only the `MetaStart` scene contains the player. `DontDestroyOnLoad` keeps it alive when a scenario scene is loaded, + so scenario scenes must **not** contain their own player or camera. +- When the role or scene changes, `PlayerSpawnManager` moves the player to the role's spawn point: a GameObject whose + name matches the role's `spawnPosition` in the [scene definition](adding_scenes.md). Afterwards + `MenuRequestManager` opens the social menu. + +## Structure + +| Object | Contents | +|---|---| +| `MetaPlayer2.0` (root) | `OVRPlayerController` + `CharacterController` (thumbstick movement, 45° snap turn, gravity), `PlayerSpawnManager`, `MenuRequestManager`, `DontDestroyOnLoad`, `DistanceObjectDesktopUser` | +| `[BuildingBlock] Camera Rig` | `OVRCameraRig` and `OVRManager`: head, eye and hand anchors | +| `[BuildingBlock] Interaction` | Interaction SDK rig: hand and controller interactions (grab, distance grab, poke, ray), synthetic hands, the `Locomotor` (teleport arc, turning, step locomotion with hands) and the eye-gaze ray (`EyeGazePointerPose`) | +| `Meta Avatar Input (OVR)` | `MetaAvatarInput`: feeds head and hand tracking into the local Meta avatar | +| `BlurEffect` | Blurred vision for roles with the `blur` restriction | + +## Movement scripts +All in `Assets/Lab/Core/Runtime/Scripts/Movement`: + +| Script | Purpose | +|---|---| +| `PlayerSpawnManager` | Moves the local player to the role's spawn point after a role or scene change | +| `MovementLock` | Locks movement while the player's status is *waiting* and lifts the `blur` restriction meanwhile | +| `TeleportOnTrigger` | Teleports a player to a target when they enter a trigger collider | +| `ContinousMovementEventBroadcaster` | Turns a thumbstick axis into continuous locomotion events | + +## Input +The project uses the Input System only (*Player Settings > Active Input Handling*). For buttons that should work with +controllers, hand tracking and a desktop keyboard alike, use `VaSiLi.Helper.LabInput` (e.g. `LabInput.PrimaryDown`). +For controller- or hand-specific input use `OVRInput` and `OVRHand`, see +[Meta input actions](https://developers.meta.com/horizon/documentation/unity/unity-inputactions). + +## Adding interactions to objects +The player already has all interactors, so an object only needs an interactable: + +1. Select the GameObject, right-click and choose *Interaction SDK* and the interaction you want (hand grab, distance + grab, poke, ray grab, ...). +2. If the object has no Collider or Rigidbody yet, click *Create* in the dialog. This adds an `ISDK_...` child object. + +Don't put every interaction type on one object. If only part of a large object should be interactable, add the +interactable to that part only. More: [Interaction SDK overview](https://developers.meta.com/horizon/documentation/unity/unity-isdk-interaction-sdk-overview/). + +## Teleporting in a scene +The `Locomotor` handles teleporting and turning. To make a floor teleportable, add a teleport interactable to it, for +example with *Add Teleport Quick Action* from the Interaction SDK, or copy `HotspotVoidFloor` and +`NavMeshHotspotWalkable` from Meta's locomotion sample scene (and `CollidersTeleportBlocker` to stop teleports through +walls). You may need to adjust the floor's height so it doesn't block the arc. + +## References +- [Meta Interaction SDK](https://developers.meta.com/horizon/documentation/unity/unity-isdk-interaction-sdk-overview/) +- [Hand tracking](https://developers.meta.com/horizon/documentation/unity/unity-handtracking-overview/) diff --git a/Documentation/docs/getting_started/setting_up.md b/Documentation/docs/getting_started/setting_up.md index 268bbe79..0e209c44 100644 --- a/Documentation/docs/getting_started/setting_up.md +++ b/Documentation/docs/getting_started/setting_up.md @@ -6,42 +6,57 @@ `git clone https://github.com/texttechnologylab/Va.Si.Li-Lab.git` ### Open the Project with UnityHub -Unity Hub downloads the correct Unity Version for you. -Add Android Support, if you want to build for Oculus VR Devices. +Unity Hub downloads the correct Unity version (6000.3) for you. +Add Android Build Support if you want to build for Meta Quest headsets. +### Import the required samples +Va.Si.Li-Lab uses code and prefabs from two package samples. Import them in *Window > Package Manager* (select the +package, *Samples* tab, *Import*). Until then you get compile errors such as `SampleAvatarEntity could not be found`. -## Import as Unity Package (Work in Progress) +| Package | Sample | +|---|---| +| Meta Avatars SDK | Sample Scenes | +| Ubiq | Demo (XRI) | -### Unity -Install Ubiq as a git package in the Unity Package Manager -``` -https://github.com/UCL-VR/ubiq.git#upm-unity-v1.0.0-pre.14 -``` -Install Va.Si.Li-Lab -``` -https://github.com/texttechnologylab/Va.Si.Li-Lab.git#upm +The Meta Avatars sample doesn't compile against Platform SDK 207 (`'CAPI' does not exist in the namespace +'Oculus.Platform'`). In `Assets/Samples/Meta Avatars SDK/40.0.1/Sample Scenes/Scripts/Utility/AvatarEditorUtils.cs` +replace the line `var result = new Request<…>(Oculus.Platform.CAPI.ovr_Avatar_LaunchAvatarEditor((IntPtr)options));` +with: + +```csharp +var result = Oculus.Platform.Avatar.LaunchAvatarEditor(options); ``` -Import Meta Avatar Samples from the Meta Avatars SDK(UPM). +Also import *TextMesh Pro Essential Resources* when Unity asks. -Fix all remaining issues listed in the Meta Project Setup Tools. +### Quick-start +1. Run *Tools > Va.Si.Li > Create Endpoints Config*. This creates `Assets/Resources/VasiliEndpoints.asset`, which holds + the addresses of all servers (Ubiq server, Database API, speech, chatbot, WebRTC). Point it to your own + [server installation](../server/server_structure.md). For quick tests, set *Ubiq Server* to the + `Nexus Connection Definition` provided by UCL (features that need our backend will be missing). +2. Open the `MetaStart` scene in `Assets/Lab/Core/Samples/Scenes` and press Play. Without a headset, use the + Meta XR Simulator. -(WIP) Ensure Active Input Handling(Player Settings) is set to Both. -(WIP) Navigate to Packages/VaSiLi-Lab/Runtime/Scenes/MetaStart or LocalLoopbackTest and open the scene. -For now only the MetaXRSimulator or a VR headset is supported. - +The event system ([Va.Si.Li-ES](../systems.md#vasili-es)) is a separate, optional package: +`"org.texttechnologylab.vasili-es": "https://github.com/texttechnologylab/Va.Si.Li-Lab.git#upm-eventsystem"`. +All other dependencies (Meta XR SDKs, XR Interaction Toolkit, WebRTC, ...) are installed automatically. +Then: +1. Import the [required samples](#import-the-required-samples) and apply the `AvatarEditorUtils.cs` fix. +2. Import the Va.Si.Li-Lab samples (*Package Manager > Va.Si.Li-Lab > Samples*) for the [scenes](special_scenes.md). +3. Fix the remaining issues listed in *Meta XR Project Setup Tool*. +4. Set *Player Settings > Active Input Handling* to *Input System Package (New)*. +5. Continue with the [quick-start](#quick-start). diff --git a/Documentation/docs/getting_started/special_scenes.md b/Documentation/docs/getting_started/special_scenes.md index ae3ac4bc..d8e72d23 100644 --- a/Documentation/docs/getting_started/special_scenes.md +++ b/Documentation/docs/getting_started/special_scenes.md @@ -1,20 +1,27 @@ # Special Scenes +The sample scenes are in `Assets/Lab/Core/Samples/Scenes`. When Va.Si.Li-Lab is installed as a package, import them +via *Package Manager > Va.Si.Li-Lab > Samples*. ## MetaStart -Starting Scene for every experiment. -Here the user can select an avatar and join a scenario. +Starting scene for every experiment. It contains the [player](player_controller.md), the scene selector and the +network connection. Here the user selects an avatar and joins a scenario. ## MetaReturn -Helping scene, when disconnecting from a scenario. -The user returns here, when he disconnects from a scenario, not to MetaStart. -Some Scripts are deleted here, that otherwise would cause problems. - -## Start -The Starting scene, using Ubiq Avatars. **Deprecated.** +Helper scene the user returns to when disconnecting from a scenario (instead of MetaStart). +Some scripts that would otherwise cause problems are removed here. ## Tutorial -Tutorial scene to get familiar with the controls. -Can be started from MetaStart. +Tutorial scene to get familiar with the controls. Can be started from MetaStart. + +## Example +Minimal scenario scene (networking and a box of pens) to copy when [adding a scene](adding_scenes.md). + +## InterViewStart, InterView, InterViewReturn +The scenes of [InterView](../systems.md#interview), used instead of MetaStart/MetaReturn for VR interviews: + +- `InterViewStart`: the participant enters the interview token (`InterviewTokenManager`). +- `InterView`: the interview room with the interviewee's seat, a mirror (self-view), the in-world browser for the + questionnaire and the WebRTC (Janus) connection to the interviewer's browser. +- `InterViewReturn`: shown after the interview. -## TestScenes/LocalLoopback Test -Test scene for local loopback testing > NetworkCapabilities. +See [InterView VR Client](interview_client.md) for their configuration. diff --git a/Documentation/docs/images/logo/vasili-lab.svg b/Documentation/docs/images/logo/vasili-lab.svg new file mode 100644 index 00000000..a127a296 --- /dev/null +++ b/Documentation/docs/images/logo/vasili-lab.svg @@ -0,0 +1,216 @@ + + + + AAAWgmp1bWIAAAAeanVtZGMycGEAEQAQgAAAqgA4m3EDYzJwYQAAABZcanVtYgAAAEdqdW1kYzJtYQARABCAAACqADibcQN1cm46YzJwYTpkMmQ4YmExYy1kY2YzLTRiNGUtYmViMy1lMzBlMGVlYWVkMmIAAAADl2p1bWIAAAApanVtZGMyYXMAEQAQgAAAqgA4m3EDYzJwYS5hc3NlcnRpb25zAAAAALxqdW1iAAAARGp1bWRjYm9yABEAEIAAAKoAOJtxE2MycGEuaW5ncmVkaWVudC52MwAAAAAYYzJzaM114EIx/uddSgUJjgYNmTUAAABwY2JvcqNpZGM6Zm9ybWF0bWltYWdlL3N2Zyt4bWxqaW5zdGFuY2VJRHgseG1wOmlpZDphYWFiODg3OS1mZGNmLTQyZDAtOWY1ZC1kZWNkYzg1MzIxM2VscmVsYXRpb25zaGlwaHBhcmVudE9mAAAB4mp1bWIAAABBanVtZGNib3IAEQAQgAAAqgA4m3ETYzJwYS5hY3Rpb25zLnYyAAAAABhjMnNoOvtyo99fczuY0Rl5t+dFNQAAAZljYm9yomdhY3Rpb25zgqJmYWN0aW9ua2MycGEub3BlbmVkanBhcmFtZXRlcnOha2luZ3JlZGllbnRzgaJjdXJseC1zZWxmI2p1bWJmPWMycGEuYXNzZXJ0aW9ucy9jMnBhLmluZ3JlZGllbnQudjNkaGFzaFggARfTJlEJzGaSLQTQc0xD0K4wyROqbZ2QZvz8noP9OGCkZmFjdGlvbngdY29tLmFudGhyb3BpYy5jbGF1ZGUucHJvdmlkZWRqcGFyYW1ldGVyc6F4H2NvbS5hbnRocm9waWMub3JpZ2luLWNvbmZpZGVuY2VndW5rbm93bmtkZXNjcmlwdGlvbnhmQ2xhdWRlIHByb3ZpZGVkIHRoaXMgZmlsZSBhdCB0aGUgcmVxdWVzdCBvZiBhIHVzZXIgYW5kIG1heSBoYXZlIGNyZWF0ZWQgb3IgbW9kaWZpZWQgdGhlIGZpbGUgY29udGVudHMubXNvZnR3YXJlQWdlbnShZG5hbWVmQ2xhdWRlcmFsbEFjdGlvbnNJbmNsdWRlZPUAAADIanVtYgAAAEBqdW1kY2JvcgARABCAAACqADibcRNjMnBhLmhhc2guZGF0YQAAAAAYYzJzaO9wNInPpRo5pHQtb7T4ZGYAAACAY2JvcqVjYWxnZnNoYTI1NmNwYWRMAAAAAAAAAAAAAAAAZGhhc2hYIMoJpbJT9j06su5kOQseJa9VaSd3+7TP8QsiO4TbljFcZG5hbWVuanVtYmYgbWFuaWZlc3RqZXhjbHVzaW9uc4GiZXN0YXJ0GQGeZmxlbmd0aBkeBAAAAj5qdW1iAAAAJ2p1bWRjMmNsABEAEIAAAKoAOJtxA2MycGEuY2xhaW0udjIAAAACD2Nib3KlY2FsZ2ZzaGEyNTZpc2lnbmF0dXJleE1zZWxmI2p1bWJmPS9jMnBhL3VybjpjMnBhOmQyZDhiYTFjLWRjZjMtNGI0ZS1iZWIzLWUzMGUwZWVhZWQyYi9jMnBhLnNpZ25hdHVyZWppbnN0YW5jZUlEeCx4bXA6aWlkOjNjMDc4N2FlLWZiZmItNDExNS05Nzc5LTJkZmNhYWY3OGI0YXJjcmVhdGVkX2Fzc2VydGlvbnODomN1cmx4LXNlbGYjanVtYmY9YzJwYS5hc3NlcnRpb25zL2MycGEuaW5ncmVkaWVudC52M2RoYXNoWCABF9MmUQnMZpItBNBzTEPQrjDJE6ptnZBm/Pyeg/04YKJjdXJseCpzZWxmI2p1bWJmPWMycGEuYXNzZXJ0aW9ucy9jMnBhLmFjdGlvbnMudjJkaGFzaFggNSg4QvP4R3scx0EuAzRtnpVPKK9QfqFCkqfENpe7CjKiY3VybHgpc2VsZiNqdW1iZj1jMnBhLmFzc2VydGlvbnMvYzJwYS5oYXNoLmRhdGFkaGFzaFgggK+3tBrZtdLaV89RdoEI3+mIQX32AcTg/etjiJxG0p90Y2xhaW1fZ2VuZXJhdG9yX2luZm+jZG5hbWVvQW50aHJvcGljIEZpbGVzZ3ZlcnNpb25lMS4wLjBrc3BlY1ZlcnNpb25lMi40LjAAABA4anVtYgAAAChqdW1kYzJjcwARABCAAACqADibcQNjMnBhLnNpZ25hdHVyZQAAABAIY2JvctKEWQISogEmGCFZAgowggIGMIIBjaADAgECAhRA5aAK7sI50L64g/oGQgU9Z1UTADAKBggqhkjOPQQDAzBJMRcwFQYDVQQKEw5BbnRocm9waWMsIFBCQzEuMCwGA1UEAxMlQW50aHJvcGljIENvbnRlbnQgQ3JlZGVudGlhbHMgUm9vdCBDQTAeFw0yNjA4MDcxODQzNTZaFw0yODA4MDYxOTQzNTZaMEQxFzAVBgNVBAoTDkFudGhyb3BpYywgUEJDMSkwJwYDVQQDEyBBbnRocm9waWMgQ2xhdWRlIENvbnRlbnQgU2lnbmluZzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJh6CmvLUBgFFNU0vUKlOVtE6djd17L5SuwX0LemFisBM3dkd/3cyjxFA3Qo5S46fX0/ihY0VZ7mfb9KF703t5OjWDBWMA4GA1UdDwEB/wQEAwIHgDAVBgNVHSUEDjAMBgorBgEEAYPoXgIBMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUzlHiBIFOZFsj+OPEz5o+nMHXXMIwCgYIKoZIzj0EAwMDZwAwZAIwMXMdFJ4BetLLVY7ORuE9noqbbAZOZn/aArXyTwFAZfKrPzxF2vPoJNf1+UCdg1XGAjBwX1zd9WGqYkqmL5SFqw1QySjr1zJfpJM9+1rdDwSPLMOPOjKuiXjoU/pUUeG9RwmhY3BhZFkNngAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPZYQOucfH7jWY0Hf5sF5zvflqdwbU5QYhDCWnScFk0zuRrJzMHxhUajibTa+ds8KJCAaf305w0HUoKnOkABkhShEYw= + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + V + a. + S + i. + L + i-Lab + + + diff --git a/Documentation/docs/images/projects/FACES.png b/Documentation/docs/images/projects/FACES.png new file mode 100644 index 00000000..347706c7 Binary files /dev/null and b/Documentation/docs/images/projects/FACES.png differ diff --git a/Documentation/docs/images/projects/NewDataSpaces.png b/Documentation/docs/images/projects/NewDataSpaces.png index a951153f..34e4903d 100644 Binary files a/Documentation/docs/images/projects/NewDataSpaces.png and b/Documentation/docs/images/projects/NewDataSpaces.png differ diff --git a/Documentation/docs/index.md b/Documentation/docs/index.md index b79b1d5f..d9b21c60 100644 --- a/Documentation/docs/index.md +++ b/Documentation/docs/index.md @@ -1,22 +1,30 @@ # Va.Si.Li-Lab +![Va.Si.Li-Lab](images/logo/vasili-lab.svg){ width="420" } + ## Description a **V**R-L**a**b for **Si**mulation-based **L**earn**i**ng. +Va.Si.Li-Lab is a Unity framework for collaborative, multi-user VR with customisable avatars and environments. +It was established in [DigiTeLL](projects/digitell.md) and is used in several [projects](projects/index.md). +Systems built on it, such as [InterView](systems.md#interview) for VR survey interviews, are listed under +[Systems](systems.md). + ## Disclaimer This documentation is work in progress. -And not all functions discribed here are implemented yet. +And not all functions described here are implemented yet. + +## Core Team +Current members of the [Text Technology Lab](https://www.texttechnologylab.org/team/) working on Va.Si.Li-Lab: -## Team -* Prof. Dr. Alexander Mehler (Leader) [:fontawesome-brands-github:](https://github.com/amehler) [:fontawesome-brands-researchgate:](https://www.researchgate.net/profile/Alexander-Mehler-2) +* Prof. Dr. Alexander Mehler (Leader) [:fontawesome-brands-github:](https://github.com/amehler) [:fontawesome-brands-researchgate:](https://www.researchgate.net/profile/Alexander-Mehler-2) * Giuseppe Abrami [:fontawesome-brands-github:](https://github.com/abrami) [:fontawesome-brands-researchgate:](https://www.researchgate.net/profile/Giuseppe-Abrami) * Mevlüt Bagci [:fontawesome-brands-github:](https://github.com/mevbagci) [:fontawesome-brands-researchgate:](https://www.researchgate.net/profile/Mevluet-Bagci) * Dr. Alexander Henlein [:fontawesome-brands-github:](https://github.com/henlein) [:fontawesome-brands-researchgate:](https://www.researchgate.net/profile/Alexander-Henlein) * Patrick Schrottenbacher [:fontawesome-brands-github:](https://github.com/TheBv) [:fontawesome-brands-researchgate:](https://www.researchgate.net/profile/Patrick-Schrottenbacher) -* Atef Ezziddin [:fontawesome-brands-github:](https://github.com/AtefEzziddin) ## Citation Please see the [publications](publications.md) for further details. @@ -27,11 +35,20 @@ Please see the [publications](publications.md) for further details. - [x] Init Documentation - [ ] Current Documentation State * [x] Home - * [ ] Getting Started + * [x] Getting Started + * [x] Modules (Endpoints, Event System, Bots, Logging) * [ ] Other Functions - * [X] Replay Experiment - * [ ] Bots - * [ ] Scenarios + * [ ] Replay Experiment (new version in preparation) + * [ ] Experiment Visualization (new version in preparation) + * [x] Scenarios * [ ] Server Structure * [x] Projects - * [x] Publications \ No newline at end of file + * [x] Systems + * [x] Publications + +## Contributors +Generated from the [GitHub contributors](https://github.com/texttechnologylab/Va.Si.Li-Lab/graphs/contributors) of the repository. + + + Contributors + diff --git a/Documentation/docs/modules/bots.md b/Documentation/docs/modules/bots.md new file mode 100644 index 00000000..288bdc95 --- /dev/null +++ b/Documentation/docs/modules/bots.md @@ -0,0 +1,57 @@ +# Bots + +Va.Si.Li-Lab has LLM-based agents that users talk to by voice and that can act in the scene through *tools*. The code +is in `Assets/Lab/Core/Runtime/Scripts/LLM` (namespace `VaSiLi.Llm`). The agents need two backend services: +[vr-agent](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/vr_agent/) (`agent` in the [endpoints](endpoints.md)), which talks to the LLM, and +[tts-stt](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/tts_stt/) (`speechSynthesis`) for speech in and out. + +## Prefabs +Both are samples (`Assets/Lab/Core/Samples/Objects/Prefabs`): + +| Prefab | Contents | +|---|---| +| `AgentToolkit` | One per scene: `AgentManager` (creates the agents), `AgentToolProvider` (collects the tools), `PlayerMicListener` (push-to-talk) and `ToolExamples` | +| `AgentBot` | One agent: the bot model, `AgentInstance`, `ChatAPI` (speech in and out) and its UI (`AgentActivatorUI`, `ToolWhitelistUI`, `ToolLogUI`) | + +`MetaStart` contains an `AgentToolkit`; `AgentManager` spawns `AgentBot` instances from it. + +## How it works +1. `AgentManager` creates the agents listed in its *Agents* field: for each it instantiates the `AgentBot` prefab at the + given position and registers the agent with the agent service (`/createAgent`). The system prompt is the agent's + *Persona* followed by the tool-call instructions. Set the list on the `AgentToolkit` instance of each scene. +2. The user activates an agent (`AgentActivatorUI`) and talks to it. `PlayerMicListener` records while *A* (or the + right index pinch, or *V* on a keyboard) is held and passes the recording to every active agent. It only records + while at least one agent is active. +3. `ChatAPI` transcribes the audio (`speechSynthesis/whisperx`) and sends the text to the agent (`agent/chat`). +4. The agent answers with text, which `ChatAPI` speaks (`speechSynthesis/tts`), or with a tool call. +5. `AgentInstance` runs tool calls the user has allowed for that agent (`ToolWhitelistUI`) and logs them (`ToolLogUI`). + +## Tools +A tool is any method marked with `[AgentTool]` on a `MonoBehaviour` in the scene. `AgentToolProvider` finds them by +reflection and sends their descriptions to the agent service (`/toolDescriptions`): + +```csharp +using UnityEngine; +using VaSiLi.Llm; + +public class LampTools : MonoBehaviour +{ + public Light lamp; + + [AgentTool("switch_lamp", "Call when the user wants the lamp switched on or off.")] + public string SwitchLamp(bool on) + { + lamp.enabled = on; + return on ? "The lamp is on" : "The lamp is off"; + } +} +``` + +Parameters can be `string`, `int`, `float`, `double`, `bool` and `Vector3`; for other types the agent passes the name of an +earlier tool call, and that call's return value is used. The returned string goes back to the agent as the result. `ToolExamples` has more examples (spawning prefabs, +changing colours and textures). Components that implement `IAgentContextInfo` add information about the scene to +the agent's context. + +## Chatbot module +`Assets/Lab/Core/Runtime/Scripts/Modules/Chatbot` is an older, simpler voice chatbot: `ChatBotManager` sends recorded +audio to `speech/query` and plays the answer, `ChatBotSelectButton` picks the chatbot. It uses the `speech` service. diff --git a/Documentation/docs/modules/endpoints.md b/Documentation/docs/modules/endpoints.md new file mode 100644 index 00000000..99502ff5 --- /dev/null +++ b/Documentation/docs/modules/endpoints.md @@ -0,0 +1,42 @@ +# Endpoints Configuration + +All server addresses of the client live in one asset, `VasiliEndpoints` +(`Assets/Lab/Core/Runtime/Scripts/Networking/VasiliEndpoints.cs`). It is loaded from any `Resources` folder by the +name `VasiliEndpoints`. + +The repository doesn't ship one. Create yours with *Tools > Va.Si.Li > Create Endpoints Config*, which writes +`Assets/Resources/VasiliEndpoints.asset`. Until then, every service falls back to `localhost` and a warning is +logged at start-up. + +!!! warning "Don't commit your endpoints file to a public repository" + It contains your server addresses and TURN credentials. Keep it in a private folder or repository. + +## Fields + +| Field | Default | Used by | Without it | +|---|---|---|---| +| `ubiqServer` | `127.0.0.1:8009` (TCP) | `VasiliServerConnection` on the `RoomClient` of the `Social Network Scene` prefab | No multiplayer room | +| `api` | `http://localhost:8000` | [Database API](../server/python_data_logger.md): scene definitions (`SceneManager`), [logging](logging.md), tutorials, connection status | No scenes in MetaStart, nothing is logged | +| `speech` | `http://localhost:8001` | [speech2text](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/speech/): `MicrophoneListener` (`/whisper`); [chatbot host](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/chatbot/): `ChatBotManager` (`/query`). Both are expected at this address | No speech input | +| `agent` | `http://localhost:8002` | [vr-agent](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/vr_agent/): `AgentManager`, `AgentInstance`, `AgentToolProvider`, `ChatAPI` (`/chat`), see [Bots](bots.md) | No agents | +| `speechSynthesis` | `http://localhost:8003` | [tts-stt](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/tts_stt/): `ChatAPI` transcription (`/whisperx`) and text-to-speech (`/tts`) | Agents don't hear or speak | +| `shapeNet` | `http://localhost:8004` | ShapeNet object search and download (`ShapeNetInterface`) | No ShapeNet objects | +| `eventTemplates` | `http://localhost:4567` | [Event System](eventsystem.md) templates (`/defaultTemplates`) | No saved or default event systems | +| `questionnaire` | *(empty)* | [InterView](../getting_started/interview_client.md): questionnaire page opened in the in-world browser (`InterviewTokenManager`) | Empty browser in InterView | +| `followupQuestionnaire` | *(empty)* | InterView: follow-up questionnaire the interviewer sends to the interviewee (`WebCamClient` follow-up button; the interview token is appended) | The follow-up button logs a warning and sends nothing | +| `janus` | `ws://localhost:8188` | WebRTC (Janus) for InterView and the webcam screen (`WebCamClient`) | No audio/video with the interviewer | +| `iceServers` | *(empty)* | STUN/TURN servers for WebRTC. Leave `username`/`credential` empty for STUN | WebRTC only works on the same network | + +Base URLs are written without a trailing slash. Use `wss://` for Janus behind a TLS reverse proxy. + +## In code + +```csharp +using VaSiLi.Networking; + +string url = VasiliEndpoints.Current.api + "/logging/status"; +``` + +`VasiliEndpoints.Current` is loaded once and cached. To add a new service, add a field with a `localhost` default and +a tooltip to `VasiliEndpoints`, and read it through `Current`. Don't hard-code hosts: +`Tools/Publish/check-public.sh` reports them. diff --git a/Documentation/docs/modules/eventsystem.md b/Documentation/docs/modules/eventsystem.md new file mode 100644 index 00000000..fefbdcf3 --- /dev/null +++ b/Documentation/docs/modules/eventsystem.md @@ -0,0 +1,52 @@ +# Event System (Va.Si.Li-ES) + +The event system lets users build *event systems* at run time, inside VR: **triggers** (an area is entered, an object +is used, a timer runs out) start **events** (spawn, move or delete objects, teleport players, change the weather, +start a vote or show feedback). Event systems can be saved as templates and loaded automatically when a scene starts. +It is described in the paper [Va.Si.Li-ES](../publications.md#vasili-es). + +The code is in `Assets/Lab/EventSystem` (assembly `Vasili.EventSystem`). As a Unity package it is installed +separately from the core, see [Setting Up](../getting_started/setting_up.md#import-as-unity-package). + +## Concepts + +| Concept | Interface | Implementations (`Runtime/...`) | +|---|---|---| +| Event system | `IEventSystem` | `EventSystem`: holds triggers and events in numbered groups, can be repeatable | +| Trigger | `ITrigger` | `Triggers/AreaTrigger`, `InteractionTrigger`, `TimeTrigger` | +| Event | `IEvent` | `Events/ObjectSpawningEvent`, `ObjectMovingEvent`, `ObjectDestructionEvent`, `PlayerTeleportationEvent`, `WeatherEvent`, `VotingEvent`, `FeedbackEvent`, `FollowingEventSystemEvent` (starts another event system) | +| Modifier | `IModifier` | `Modifiers/Modifier_Gravity`, `_Jump`, `_Speed`, `_Weather`, `_Zoom`, `_Map`, `_Admin`; `Modefier_Handler` switches the modifiers in its `Modifier_Container` on or off according to the scene's `ModifierSettings` | + +When all triggers of a group have fired, the events of that group run. Group `0` is shared: its triggers are required +by every group, and its events run whenever any group fires. A repeatable event system resets its triggers afterwards, +otherwise it deletes itself. Triggers and events serialise themselves to JSON (`createJson`), which is how templates +are stored. + +## Building blocks + +| Prefab | Purpose | +|---|---| +| `Manager/EventSystemManager` | Menu that lists the scene's event systems, loads templates and the scene's default event systems on start | +| `Factory/EventSystemFactory` | Creates one event system with its triggers and events | +| `MarkerTool/MarkerTool` | Tool for placing triggers and events in the scene | +| `EnvironmentSystem/Weather/Weather_Controller` | Weather (rain, snow, thunderstorm) used by `WeatherEvent` and `Modifier_Weather` | +| `VotingSystem/Voting_Controller` | Votes (yes/no or single choice) used by `VotingEvent` | +| `FeedbackSystem/Notification Manager`, `FeedbackWindow` | Notifications and feedback used by `FeedbackEvent` | +| `EventSystemPrefabs/*` | Ready-made items: binoculars, jump and sprint shoes, map, weather clouds, environment changer | + +To use it in a scene, add the `EventSystemManager` prefab (and the controllers for the events you need) to the scene. + +## Templates +`EventSystemManager` reads and writes templates at `eventTemplates + "/defaultTemplates"` (see +[Endpoints](endpoints.md)). A template belongs to the scene whose name it stores; templates of the current scene are +offered in the template panel, and default ones are created when the scene loads. + +## Extending +- **New trigger:** a `MonoBehaviour` implementing `ITrigger`. Call `notifySystem()` when it fires, and return its + state from `getState()`. +- **New event:** a `MonoBehaviour` implementing `IEvent`. Do the work in `trigger()`, show what will happen in + `previewOn()`/`previewOff()`. +- **New modifier:** implement `IModifier` and add it to the `Modifier_Container` used by `Modefier_Handler`. The + scene's `ModifierSettings` decide whether it is allowed and clamp its value. + +Both triggers and events need `createJson(groupID)` so they can be saved in templates. diff --git a/Documentation/docs/modules/logging.md b/Documentation/docs/modules/logging.md new file mode 100644 index 00000000..78caac75 --- /dev/null +++ b/Documentation/docs/modules/logging.md @@ -0,0 +1,37 @@ +# Logging + +Va.Si.Li-Lab records what happens in a session (movement, gaze, faces, hands, interactions, audio) and sends it to +the [Database API](../server/python_data_logger.md), which stores it in [MongoDB](../server/mongodb.md). The data +model behind it is described in +[A Multimodal Data Model for Simulation-Based Learning with Va.Si.Li-Lab](../publications.md#mehler2023). + +The code is in `Assets/Lab/Core/Runtime/Scripts/Logging`. The address comes from `api` in the +[endpoints configuration](endpoints.md). + +## MetaDataLogger +`MetaDataLogger` on the `Social Network Scene` prefab collects the data of the local player and sends it in batches: + +| Route (`api` + ...) | Content | +|---|---| +| `/logging/playerLogIn` | A player joins | +| `/logging/playerRoleLogIn` | A player takes a role | +| `/logging/levelChange` | The admin changes the level | +| `/logging/player` | Head, hand and body tracking; with Meta avatars also eyes, face and fingers (`metaMessage`) | +| `/logging/object` | Positions of logged objects | +| `/logging/special` | Button presses and spawned objects | +| `/logging/logMisc` | Other events | + +The message types are in `DataLoggerTypes.cs`. How the API splits them into collections is described in the +[Database API documentation](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/database_api/). + +!!! warning "Logging failures don't stop a session" + If the API can't be reached, the session continues normally and nothing is stored. Check that data arrives + (e.g. `api + /logging/status`, shown by `ConnectionStatusMonitor`) before a study session starts. + +## Other loggers +| Component | Purpose | +|---|---| +| `FileLogger` | Writes the JSON messages to the local file system instead | +| `LogManager`, `Collectors/MovementLogCollector` | Collector-based logging (`ILogCollector`, `IContinousLogCollector`, `IEventLogCollector`) | +| `LogToScreen` | Shows Unity log output in VR | +| `DevLogConfig` | Disables per-log stack traces in development builds | diff --git a/Documentation/docs/other/bots.md b/Documentation/docs/other/bots.md deleted file mode 100644 index a2d24672..00000000 --- a/Documentation/docs/other/bots.md +++ /dev/null @@ -1,2 +0,0 @@ -# Bots -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/other/experiment_visualization.md b/Documentation/docs/other/experiment_visualization.md new file mode 100644 index 00000000..f49267a0 --- /dev/null +++ b/Documentation/docs/other/experiment_visualization.md @@ -0,0 +1,6 @@ +# Experiment Visualization + +!!! info "In preparation" + A new tool for analysing and visualising recorded experiments (audio transcription, tracking data) is in + preparation and will be documented here once it is released. The previous Streamlit tools are no longer + maintained. diff --git a/Documentation/docs/other/experiment_visualizer_audio.md b/Documentation/docs/other/experiment_visualizer_audio.md deleted file mode 100644 index bb08f066..00000000 --- a/Documentation/docs/other/experiment_visualizer_audio.md +++ /dev/null @@ -1,102 +0,0 @@ -# Experiment Visualization Audio - -## Introduction - -This tool enables the transcription of voice recordings made of participants in experiments. A sentiment analysis is automatically included in the visualisation. Additionally, users may highlight tags that they have defined. Each turn can be analysed by a LLM, and the prompt used can be customised. There is a default prompt with predefined types of turns. Furthermore, users can generate a wordcloud of the transcription of the selected scenario. - -## Setup - -### Requirements - -- Python 3.12.2 -- login.json in directory ../db/login/login.json (Login data for MongoDB) - -#### Packages -- streamlit (1.38.0) [(more information)](https://docs.streamlit.io/get-started/installation) -- pymongo (4.8.0) [(more information)](https://pymongo.readthedocs.io/en/stable/installation.html) -- pydub (0.25.1) [(more information)](https://github.com/jiaaro/pydub) -- numpy (1.26.4) [(more information)](https://numpy.org/install/) -- openai-whisper (20231117) [(more information)](https://github.com/openai/whisper) -- spacy (3.7.6) [(more information)](https://spacy.io/usage) -- wordcloud (1.9.3) [(more information)](https://github.com/amueller/word_cloud) -- matplotlib (3.9.2) [(more information)](https://matplotlib.org/stable/) -- textblob (0.18.0.post0) [(more information)](https://textblob.readthedocs.io/en/dev/) -- transformers (4.44.2) [(more information)](https://github.com/huggingface/transformers) -- torch (2.4.0) [(more information)](https://pytorch.org/) - -### Getting started - -To start enter "streamlit run Live_Experiment_Analysis.py" in terminal. - -## Pages - -### Audiodata Analysis - -#### Load Data - -On this page you can initialize the transcription and see the visualisation. - -The first step is to select the scenario and the date from which the voice recordings are to be analyzed. -This tool uses Whisper to transcribe the audio data. There are different modelsizes of whisper available. Therefore, in the next step, select one of the modelsizes from the dropdown menu. -Once you have selected the scenario, date and model size, press the button **load data** to - -1. Export documents from the MongoDB "Audio" database with the selected scene and date. -2. Convert the audio data into a .wav file and export it. -3. Transcribe the audio files using the whisper model of your choice. - -The button **load turntaking analysis** allows you to load a turntaking analysis using the llm "NousResearch/Llama-2-7b-chat-hf" from [Huggingface](https://huggingface.co/NousResearch/Llama-2-7b-chat-hf). The prompt that the LLM takes as input can be edited in the [Settings](#edit-prompt-for-turntaking-analysis). - -If you have previously loaded the turntaking analysis and wish to reload it (e.g. after editing the prompt), you can press the **reload turntaking analysis** button. - -#### Wordcloud - -By pressing the button **show wordcloud**, you can create a wordcloud of the full transcription of the scenario. Words contained in the german_stopwords_full.txt file are excluded. - -#### Show Transcription and Turntaking Analysis - -Use the slider bar to set the interval at which the transcription will be displayed. - -You have the option to highlight personalised tags in the transcription. Tags can be added or deleted in the [Settings](#set-tags). To highlight your tags, make sure the corresponding checkbox is selected. - -Finally, you can visualise the voice recordings as a messenger by clicking on either **show transcription** or **show transcription with turntaking analysis**. To view the transcription with a turntaking analysis, you must first click on **load turntaking analysis**. - -The visualisation of the speech recordings also includes a sentiment analysis. The sentiment of each sentence is colour coded in five categories: negative (red), tendence negative (orange), neutral (grey), tendence positive (blue) and positive (green). The thresholds between each category can be adjusted in the [Settings](#set-thresholds-for-sentiment-analysis). - -### Settings - -This is the page where you can personalise the features of this tool according to your preferences. - -#### Set Tags - -Tags can be added using the input field on the left and removed with the input field on the right. The tags currently defined are displayed below the input fields. All of the tags that are shown below *current tags* will be highlighted if you select the corresponding option in the [Audiodata Analysis](#audiodata-analysis). - -#### Set Thresholds for Sentiment Analysis - -This tool allows you to view five different levels of sentiment: negative, tendence negative, neutral, tendence positive and positive. Each level represents an interval within the range (-1, 1). The value of the sentiment is within the interval indicated by the color of the transcription. - -- a is the threshold between "negative" and "tendence negative" -- b is the threshold between "tendence negative" and "neutral" -- c is the threshold between "neutral" and "tendence positive" -- d is the threshold between "tendence positive" and "positive" - -Note that the parameters should be selected according to a < b < c < d. Also, no parameter should be selected the same as another. -If you reload the "Settings" page, the thresholds will be set to te default values (a=-0.2, b=-0.01, c=0.01, d=0.2). - -#### Edit Prompt for Turntaking Analysis - -To analyze the turns of the speech recordings, the LLM takes a prompt describing the types of turns. -The prompt consists of two parts: one is editable and the other is not. - -The first part of the prompt can be edited using the input field *customize prompt*. - -There is a default prompt which you can see below *see full default prompt here* when you first open the "Settings" page. - -The input field is not expandable. If you want to edit a part of the default prompt, it is recommended that you copy the default prompt and edit it externally in a text editor of your choice. -You can then press *empty prompt* and paste your edited prompt into the input field. Then press Enter to submit the updated prompt. -To return to the default prompt, press *set prompt to default*. - -The following part of the prompt cannot be edited. This is to ensure, that the two parts of a turn are placed correctly within the prompt. - -``` -"Turn: Person X says: {first part of the turn} Person Y responds to Person X: {second part of the turn}"<\input> -``` diff --git a/Documentation/docs/other/experiment_visualizer_trackingdata.md b/Documentation/docs/other/experiment_visualizer_trackingdata.md deleted file mode 100644 index 3a245c02..00000000 --- a/Documentation/docs/other/experiment_visualizer_trackingdata.md +++ /dev/null @@ -1,173 +0,0 @@ -# Experiment Visualisation Trackingdata - -## Overview - -### Description -This Streamlit Web Application uses Data stored in a MongoDB from Experiments and displays multiple plots visualising the behavior of the test subjects. - -## Setup -Python version 3.12.0 -login.json in directory ../db/login/login.json (Login data for MongoDB) - -### Requirements -The Requirements.txt is in the Project - -- matplotlib==3.9.0 -- numpy==1.26.4 -- numpy_quaternion==2023.0.4 -- pandas==2.2.2 -- Pillow==10.3.0 -- plotly==5.22.0 -- pydub==0.25.1 -- pymongo==4.7.2 -- Requests==2.31.0 -- scipy==1.13.1 -- seaborn==0.13.2 -- streamlit==1.34.0 -- textblob==0.18.0.post0 -- whisper==1.1.10 - - -## How to use - -### starting the Application -In your Terminal type streamlit run streamlit run Live_Experiment_Analysis.py -In the Sidebar navigate to Trackingdata. - -### Usage - -In the Sidebar first select the experiment and the date. This will display all playerIds that where logged in that day. - -![image](https://github.com/user-attachments/assets/c6c39c4e-d952-4686-95cd-74217f3d7785) - -**Optional** You can now filter the players to only show one role -*Note** some experiments have no role or role is not stored correctly - -![image](https://github.com/user-attachments/assets/8bd4ec7c-7e9c-4433-a0bb-14e25197131e) - -Now select the playerIds. This will first load the level selector. After Loading finished you can click the "Load Plots" button. -When a player is first loaded data from the mongodb has to be fetched, which will take some time. After the first time no data has to be fetched again (only session wise). - -At the top you can switch between the Plots and Multimodal chat Tabs - -### Options -In the options Menu you have the ability to select different settings. -**You have to click submit to save the changes** -Starting from left to right. -- Select filtered Objects which will not be displayed in the Interaction Heatmap -- Select which Eye should be used to calclualte the Eygazes Plot. Default is 0 = left Eye -- Select smoothing factor for the Linegraphs. default is 250 lower number means less smoothing -- Select the Radius for the Head and Body Sphere (Hitbox) (When calculating if a person looked at another person) -- Checkbox display contious plots. Displays a continous motion of the facial expression polygons on the avatarface (Default is False because it does take a lot of time especially for the longer experiments to go through all players) -- checkbox show teleportations. Shows in the linegraphs when a player is teleportated (skews the graph) -- checkbox less expensive plots. Doesn't show facial expressions and the heatmap. The heatmap is what really takes a long time to calculate - -![image](https://github.com/user-attachments/assets/30ad7c08-8d08-403f-8d2e-da754c56fa46) - -### Levelselector -**You have to click submit to save the changes** -**Note you also have to tick the checkbox to filter out the time** -In the selectboxes you can see each level when it started for each player. -Then in the slider choose the time you want to see -The Interactionheatmap, Facial expressions and the multimodal chat is nopt influenced by the levelfilter - -![image](https://github.com/user-attachments/assets/49099bca-31ca-47f9-9026-99b343dd00f5) - -## Functions - -### What are the plots representing -- Eyegaze: The orange dot is the position of the Eye and the orange line is the straight forward looking direction. -The blue points represent where the person looked originating from [0, 0, 0] (orange dot) - -![image](https://github.com/user-attachments/assets/29fc171d-a5a4-45b2-b2ec-aa6c69230a33) - -- Handheatmap: The orange dot is the Position of the Head. The "Volumeheatmap" shows where relatively to the head the hand is. - **doesn't work as intended** The handposition is rotated to the direction which the **Body** faces, which causes the heatmap to be displayed "around" the player. - extra eklärung - -- Facial Expressions: Displays the intensity of movement for 63 key points on the face, indicating how much each part has moved. see project files for Legend - - Using this scale purple being the lowest and yellow the highest - -![image](https://github.com/user-attachments/assets/db7c8a18-82a9-43b5-b711-e1eef246c1e9) - -- Interaction heatmap: Shows where and how the player has interacted with an object on the map. (for some experiments are the sizes of the icons skewed because some experiments have different dimensions) - -- Heatmap: Shows a heatmap for positional points - -![image](https://github.com/user-attachments/assets/df35e730-e159-45f5-ada7-a60ac662e35d) - -If there is no no backroundmap the heatmap will be displayed on a white backround -![image](https://github.com/user-attachments/assets/3d9710be-74ea-44c5-9a86-5629a6639c26) - - -- Multimodal chat: Displays a timeline from top (start) to bottom (end), whether a player was looking at another player's head or body and when a player interacted with an object. - -### structure of data - -For all plots (except facial expressions) was first the data fetched and calculated upon and then with this data where the plots made. Some data is used multiple times for different plots. - -When the data is fetched every player is given a list where his data is stored [ [], [], [] ] - -See input output for how _fetch_data works - -Please see in project file /datastructure/body for List_of_every_player_body explanation diagramm - -list_of_every_player_head and list_of_every_player_hand follows a similiar formula - - - -## input Ouput - -Not all function are here, because some are very specific. - -### linegraph.py - -#### _fetch_data: -takes in( a list of playerids, the collection to search in, what to search for ) # playerId is always returned - - ["324Fg", "jshfu"], main.collection_body, {serverTime:1, position:1} - - Output: returns a list of lists [ [], [] ] where each list is for one player, in which the data is stored - -### Eye_tracking.py -#### create_quaternion(orientation): - -takes in a "quaternion" in a "list format" [0, 0, 0, 1] with w being the last element. - -output: a quaternion in matrix format - -### facial_expression.py -#### avatarId_finder(playerId): - -takes in **one** playerid as a string - -outputs: an int 0 to 31 representing the avatarId of the playerId - - -#### get_intensitity_for_expression(percent): -Input: Float number 0 to 1 (a percent) -Output: 4 tuple RGBA in normalized format (r,g,b,opacity) - -### interactions.py - -#### interactionShapeNetObjectAdder(interactionsObjects): -**Only for Shapenet objects if no shapenet object return the same dict** - -Input: a dictionary of interactions done {'objectName': '343434335353_center', 'hand': 'Right', 'interaction': 'ungrasped', 'playerId': '520708ed-59e733b7', 'server-timestamp': datetime.datetime(2024, 5, 29, 9, 34, 30), 'position': {'x': -821.1394, 'y': 51.47187, 'z': -770.7883}} -Output: return the dict with the objectName changed to the real Objectname + adds two new key values pairs. the old Id name and categories - -### levelSelecter.py - -#### getMinMaxTime(playerIds): -Input : list of playerIds -Output: **Across all playerIds** that were given outputs: a list of a dictionary with the min and max of the server time [ { maxServerTime: } ] -minServerTime first Servertime and "maxServerTime" last ServerTime (Login, Logoff) - -### multimodalChat.py - -#### collisionChecker(personPosition, personViewPosition, secondPersonSphere, radius=0.15): -Input: Position first point [ 0, 4, 3 ] , position second Point, PositionSphere, **optional radius=float** -Output: weather or not the line beetween the first two points, intersects the third position (Sphere) with the radius= - - diff --git a/Documentation/docs/other/replay.md b/Documentation/docs/other/replay.md index 478b4509..74b09f90 100644 --- a/Documentation/docs/other/replay.md +++ b/Documentation/docs/other/replay.md @@ -1,22 +1,26 @@ # Simulations & Experiments Playback +!!! warning "Not available in the current version" + The `SimulationLoader` scene and the `Assets/Simulation_dev` folder described here are not part of the current + repository. A new replay tool is in preparation; until then this page documents the earlier one. + Simulations and Experiments performed using this project can be replayed using the following steps: ## Add Scenes to Build Setting -Scenes to be replayed and the Simulaton loader scene should be added to build setting. +Scenes to be replayed and the SimulationLoader scene should be added to build setting. -1. Open `SmulatonLoader` scene located in `Assets/Simulation_dev/Scenes`. +1. Open `SimulationLoader` scene located in `Assets/Simulation_dev/Scenes`. 2. From `File` tab choose `Build Settings...`. -3. Add `SimulatoinLoader` scene to build by clickng `Add Open Scenes`. -4. Make sure `SimulatioinLoader` scene is enabled using the toggle to its left side. +3. Add `SimulationLoader` scene to build by clicking `Add Open Scenes`. +4. Make sure `SimulationLoader` scene is enabled using the toggle to its left side. 5. Add and enable all scenes of the Simulations/Experiments to be replayed. ## Start Playback -To start playback open and play `SimulatonLoader` scene. +To start playback open and play `SimulationLoader` scene. ### Local Playback @@ -26,11 +30,11 @@ Choose the scene, date and players to be replayed then click on `Next`. ### Online Playback -If no local data found, or if online data is preferred, using the `Load Online Data` toggle **(Disabled by default)** will connect to the server and retreive information about Simulations and its data. +If no local data found, or if online data is preferred, using the `Load Online Data` toggle **(Disabled by default)** will connect to the server and retrieve information about Simulations and its data. -By online Playback the followng options are available: +By online Playback the following options are available: -- Keeping the chosen playbacks data if local instances of it are found, or Saving the chosen playbacks if no local data of it found using the `Keep Json Copy` toggle. Loacl data will be yellow colored by the list of players to choose from. **(Enabled by default)** +- Keeping the chosen playbacks data if local instances of it are found, or Saving the chosen playbacks if no local data of it found using the `Keep Json Copy` toggle. Local data will be yellow colored by the list of players to choose from. **(Enabled by default)** - Use online data instead of local data of the chosen playbacks, if any local data found, using the `Updated Info` toggle. This will Replace local data if `Keep Json Copy` toggle is enabled. **(Disabled by default)** - **Note**: - Disabling `Keep Json Copy` will remove any local data, **of the chosen playbacks only**, saved previously. @@ -41,9 +45,9 @@ After choosing players click on `Next`. ### Choosing Avatars -After choosng players and clicking `Next`, a new page is loaded with the name of chosen Scene, *[Instructions](#keyboard)* on how to control playback with keyboard and the ability to choose different avatars for each player. +After choosing players and clicking `Next`, a new page is loaded with the name of chosen Scene, *[Instructions](#keyboard)* on how to control playback with keyboard and the ability to choose different avatars for each player. -Default avatars will be set according to the info retreived online or from local data. If none found, Avatar Nr. 0 will be set as default. +Default avatars will be set according to the info retrieved online or from local data. If none found, Avatar Nr. 0 will be set as default. Click on `Load Scene` to Start playback after all required data have been loaded. @@ -69,7 +73,7 @@ There are several ways to control the playback: - `C`: Activate/Disable 1. Person Camera. -**Note:** While in 1. Person Camera, no camera movements controls are available. This will imitate being in the prespective of the current player. +**Note:** While in 1. Person Camera, no camera movements controls are available. This will imitate being in the perspective of the current player. #### Free Camera @@ -78,7 +82,7 @@ There are several ways to control the playback: - `T,G`: Vertical Rotation. - `F,H`: Horizontal Rotation. - `Z,X`: Height Control. -- `V,B`: Jump to nexy player's position. +- `V,B`: Jump to next player's position. **Note:** Once activated, Camera will be positioned at a default location. Depending on the scene, this location might be far from players. using `V,B` will move Camera to the position of the next/previous player. @@ -105,16 +109,16 @@ click on `Simulation(Clone)` object under `DontDestroyOnLoad`. Custom inspector - `Face/Eye Weights`: With a value between 0 and 2 to control the intensity of the Face/Eye pose for **all** playbacks. - `Hand Pos. Senset.`: With a value between 0.4 and 1.5. When the difference in vertical hand position between the current frame and the previous one is bigger than this value, -the old value from the previos frame will be taken instead and ignore the current one, to avoid having hands in extrem positions. This can happen mostly when switching +the old value from the previous frame will be taken instead and ignore the current one, to avoid having hands in extreme positions. This can happen mostly when switching between Finger tracking and Controller tracking. -- `Voice Ind. Senset.`: With a value between 0.0001 and 1. Controls the sensetivity of the voice indicator above each player's avatar. -Higher values will requier higher sounds to activate the indicator. By value 1 the voice indicator will mostly be disabled. +- `Voice Ind. Senset.`: With a value between 0.0001 and 1. Controls the sensitivity of the voice indicator above each player's avatar. +Higher values will require higher sounds to activate the indicator. By value 1 the voice indicator will mostly be disabled. #### Camera Control - `1. Person view` toggle to activate 1. Person Camera. - `Free Camera` toggle to activate Free Camera. This will show additional buttons underneath for movement control. -- `Rotation & Zoom` Controls when Free Camera is disabled / `Movament, Rotation & Height` Controls When Free Camera +- `Rotation & Zoom` Controls when Free Camera is disabled / `Movement, Rotation & Height` Controls When Free Camera is activated. #### Debug Mode @@ -124,7 +128,7 @@ is activated. ### Time & Speed Slider - `Speed Slider`: With a value between 0.1 and 2, will control the speed of all playbacks' movements and sounds. -- `Time Slider`: Once selected or changed value, playbacks' movements and sounds will be paused. Unpause through inspector or keypoard `P` to continue. +- `Time Slider`: Once selected or changed value, playbacks' movements and sounds will be paused. Unpause through inspector or keyboard `P` to continue. ## Record Playback diff --git a/Documentation/docs/projects/faces.md b/Documentation/docs/projects/faces.md index 4b869fa3..2d401663 100644 --- a/Documentation/docs/projects/faces.md +++ b/Documentation/docs/projects/faces.md @@ -6,8 +6,11 @@ ## FACES +![FACES](../images/projects/FACES.png){loading=lazy width=320} + [FACES](https://gepris.dfg.de/gepris/projekt/539621548?language=en) (Feasibility, acceptance, and data quality of new multimodal surveys) is part of this SPP. +Project website: [texttechnologylab.github.io/FACES](https://texttechnologylab.github.io/FACES/). Conventional surveys record the answer and discard everything that produced it - the hesitation, the gaze, whether the question was understood at all. FACES asks @@ -27,7 +30,10 @@ hand movement, facial expression and the answers themselves are all captured on shared clock. InterView is built on Va.Si.Li-Lab and adds a WebRTC media layer, a browser-based -interviewer console, and a token-driven session model. +interviewer console, and a token-driven session model. Avatar emotions come from +[ReEmote](../systems.md#reemote). Both are listed under +[Systems using Va.Si.Li-Lab](../systems.md), their papers under +[Publications](../publications.md#interview). !!! info "InterView has its own documentation" Architecture, the full deployment guide, and reference documentation for diff --git a/Documentation/docs/projects/index.md b/Documentation/docs/projects/index.md new file mode 100644 index 00000000..a121c0d0 --- /dev/null +++ b/Documentation/docs/projects/index.md @@ -0,0 +1,11 @@ +# Projects + +Research projects in which Va.Si.Li-Lab is developed and used. + +| Project | Funding | Va.Si.Li-Lab is used for | +|---|---|---| +| [DigiTeLL](digitell.md) | Goethe University Frankfurt, *Digital Teaching and Learning Lab* | Origin of Va.Si.Li-Lab: simulation-based learning in VR for teaching | +| [ViCom / GeMDiS](vicom.md) | DFG Priority Programme 2392 *Visual Communication* | Gestures in dialogue: VR-based multimodal data collection ([FraGA](../systems.md#fraga), [Semantic Scene Builder](../systems.md#semantic-scene-builder)) | +| [New Data Spaces / FACES](faces.md) | DFG Priority Programme 2431 *New Data Spaces* | Survey interviews in VR ([InterView](../systems.md#interview), [ReEmote](../systems.md#reemote)) | + +Systems built on Va.Si.Li-Lab are listed in [Systems using Va.Si.Li-Lab](../systems.md). diff --git a/Documentation/docs/publications.md b/Documentation/docs/publications.md index 65124dfa..18220327 100644 --- a/Documentation/docs/publications.md +++ b/Documentation/docs/publications.md @@ -1,70 +1,476 @@ -## Publications - -#### A Multimodal Data Model for Simulation-Based Learning with Va.Si.Li-Lab - -``` bibtex -@inproceedings{Mehler:et:al:2023:a, - author = {Mehler, Alexander and Bagci, Mevl{\"u}t and Henlein, Alexander - and Abrami, Giuseppe and Spiekermann, Christian and Schrottenbacher, Patrick - and Konca, Maxim and L{\"u}cking, Andy and Engel, Juliane and Quintino, Marc - and Schreiber, Jakob and Saukel, Kevin and Zlatkin-Troitschanskaia, Olga}, - address = {Cham}, - booktitle = {Digital Human Modeling and Applications in Health, Safety, Ergonomics and Risk Management}, - editor = {Duffy, Vincent G.}, - isbn = {978-3-031-35741-1}, - pages = {539--565}, - publisher = {Springer Nature Switzerland}, - title = {A Multimodal Data Model for Simulation-Based Learning with Va.Si.Li-Lab}, - year = {2023}, - doi = {10.1007/978-3-031-35741-1_39} -} -``` -[[Link]](https://link.springer.com/chapter/10.1007/978-3-031-35741-1_39) [[RG]](https://www.researchgate.net/publication/372298766_A_Multimodal_Data_Model_for_Simulation-Based_Learning_with_VaSiLi-Lab) +# Publications +Publications on Va.Si.Li-Lab and on the systems built with it. + +If you use Va.Si.Li-Lab, please cite [A Multimodal Data Model for Simulation-Based Learning with Va.Si.Li-Lab](#mehler2023). + +## Va.Si.Li-Lab + +### Va.Si.Li-Lab: A Virtual Reality Solution for Applications in the Field of Educational and Social Science Technologies + +**Schrottenbacher, Abrami, Bagci & Mehler** (2026). In *Digitale Learning Designs für die Hochschullehre gemeinschaftlich +entwickeln*, Waxmann, 212–229. +[[DOI]](https://doi.org/10.31244/9783818851170) + +??? note "BibTeX" + + ```bibtex + @incollection{Schrottenbacher:et:al:2026:b, + author = {Schrottenbacher, Patrick and Abrami, Giuseppe and Bagci, Mevl{\"u}t + and Mehler, Alexander}, + title = {{Va.Si.Li-Lab}: A Virtual Reality Solution for Applications in + the Field of Educational and Social Science Technologies}, + booktitle = {Digitale Learning Designs f{\"u}r die Hochschullehre gemeinschaftlich + entwickeln: Werkzeuge -- Ressourcen -- Szenarien}, + editor = {Zeaiter, Sabrina and Stierwald, Mona}, + publisher = {Waxmann}, + year = {2026}, + pages = {212--229}, + doi = {10.31244/9783818851170} + } + ``` + +--- + +### Simulation-Based Learning in Virtual Reality: Three Use Cases from Social Science and Technological Foundations in Terms of Va.Si.Li-Lab + +**Bagci, Mehler, Abrami, Schrottenbacher, Spiekermann, Konca, Schreiber, Saukel, Quintino & Engel** (2025). +*Technology, Knowledge and Learning*, Springer Nature. +[[DOI]](https://doi.org/10.1007/s10758-025-09837-7) + +??? note "BibTeX" + + ```bibtex + @article{Bagci:et:al:2025, + author = {Bagci, Mevl{\"u}t and Mehler, Alexander and Abrami, Giuseppe and Schrottenbacher, Patrick + and Spiekermann, Christian and Konca, Maxim and Schreiber, Jakob and Saukel, Kevin + and Quintino, Marc and Engel, Juliane}, + title = {Simulation-Based Learning in Virtual Reality: Three Use Cases + from Social Science and Technological Foundations in Terms of + Va.Si.Li-Lab}, + journal = {Technology, Knowledge and Learning}, + publisher = {Springer Nature}, + year = {2025}, + month = {April}, + day = {01}, + issn = {2211-1670}, + doi = {10.1007/s10758-025-09837-7}, + url = {https://doi.org/10.1007/s10758-025-09837-7} + } + ``` + +--- + +### Towards New Data Spaces for the Study of Multiple Documents with Va.Si.Li-Lab: A Conceptual Analysis + +**Mehler, Bagci, Schrottenbacher, Henlein, Konca, Abrami, Bönisch, Stoeckel, Spiekermann & Engel** (2024). +In *Students', Graduates' and Young Professionals' Critical Use of Online Information*, Springer, 259–303. +[[DOI]](https://doi.org/10.1007/978-3-031-69510-0_12) + +??? note "BibTeX" + + ```bibtex + @inbook{Mehler:et:al:2024:a, + author = {Mehler, Alexander and Bagci, Mevl{\"u}t and Schrottenbacher, Patrick + and Henlein, Alexander and Konca, Maxim and Abrami, Giuseppe and B{\"o}nisch, Kevin + and Stoeckel, Manuel and Spiekermann, Christian and Engel, Juliane}, + editor = {Zlatkin-Troitschanskaia, Olga and Nagel, Marie-Theres and Klose, Verena + and Mehler, Alexander}, + title = {Towards New Data Spaces for the Study of Multiple Documents with + Va.Si.Li-Lab: A Conceptual Analysis}, + booktitle = {Students', Graduates' and Young Professionals' Critical Use of + Online Information: Digital Performance Assessment and Training + within and across Domains}, + year = {2024}, + publisher = {Springer Nature Switzerland}, + address = {Cham}, + pages = {259--303}, + isbn = {978-3-031-69510-0}, + doi = {10.1007/978-3-031-69510-0_12}, + url = {https://doi.org/10.1007/978-3-031-69510-0_12}, + } + ``` + +--- + +### A Multimodal Data Model for Simulation-Based Learning with Va.Si.Li-Lab { #mehler2023 } + +**Mehler, Bagci, Henlein, Abrami, Spiekermann, Schrottenbacher, Konca, Lücking, Engel, Quintino, Schreiber, Saukel & +Zlatkin-Troitschanskaia** (2023). *Digital Human Modeling and Applications in Health, Safety, Ergonomics and Risk +Management* (HCII 2023), Springer, 539–565. +[[Link]](https://link.springer.com/chapter/10.1007/978-3-031-35741-1_39) +[[RG]](https://www.researchgate.net/publication/372298766_A_Multimodal_Data_Model_for_Simulation-Based_Learning_with_VaSiLi-Lab) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Mehler:et:al:2023:a, + address = {Cham}, + author = {Mehler, Alexander and Bagci, Mevl{\"u}t and Henlein, Alexander + and Abrami, Giuseppe and Spiekermann, Christian and Schrottenbacher, Patrick + and Konca, Maxim and L{\"u}cking, Andy and Engel, Juliane and Quintino, Marc + and Schreiber, Jakob and Saukel, Kevin and Zlatkin-Troitschanskaia, Olga}, + booktitle = {Digital Human Modeling and Applications in Health, Safety, Ergonomics + and Risk Management}, + editor = {Duffy, Vincent G.}, + isbn = {978-3-031-35741-1}, + pages = {539--565}, + publisher = {Springer Nature Switzerland}, + title = {A Multimodal Data Model for Simulation-Based Learning with Va.Si.Li-Lab}, + year = {2023}, + doi = {10.1007/978-3-031-35741-1_39}, + } + ``` + +--- + +### Va.Si.Li-Lab as a Collaborative Multi-User Annotation Tool in Virtual Reality and Its Potential Fields of Application + +**Abrami, Mehler, Bagci, Schrottenbacher, Henlein, Spiekermann, Engel & Schreiber** (2023). *Proceedings of the 34th ACM +Conference on Hypertext and Social Media* (HT '23). +[[Link]](https://dl.acm.org/doi/10.1145/3603163.3609076) [[PDF]](https://dl.acm.org/doi/pdf/10.1145/3603163.3609076) +[[RG]](https://www.researchgate.net/publication/371303212_VaSiLi-Lab_as_a_collaborative_multi-user_annotation_tool_in_virtual_reality_and_its_potential_fields_of_application) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Abrami:et:al:2023, + author = {Abrami, Giuseppe and Mehler, Alexander and Bagci, Mevl\"{u}t and Schrottenbacher, Patrick + and Henlein, Alexander and Spiekermann, Christian and Engel, Juliane + and Schreiber, Jakob}, + title = {Va.Si.Li-Lab as a Collaborative Multi-User Annotation Tool in + Virtual Reality and Its Potential Fields of Application}, + year = {2023}, + isbn = {9798400702327}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3603163.3609076}, + doi = {10.1145/3603163.3609076}, + booktitle = {Proceedings of the 34th ACM Conference on Hypertext and Social Media}, + articleno = {22}, + numpages = {9}, + location = {Rome, Italy}, + series = {HT '23}, + pdf = {https://dl.acm.org/doi/pdf/10.1145/3603163.3609076} + } + ``` + +## Systems using Va.Si.Li-Lab + +The systems are described in [Systems using Va.Si.Li-Lab](systems.md). + +### InterView: Towards a Unified VR-Capable Interview Environment { #interview } + +**Schrottenbacher, Mehler, Abrami, Kleine & Stingl** (2026). Preprint, under review at the *International Journal of +Human-Computer Studies*. +[[DOI]](https://doi.org/10.2139/ssrn.7459949) + +??? note "BibTeX" + + ```bibtex + @misc{Schrottenbacher:et:al:2026:c, + author = {Schrottenbacher, Patrick and Mehler, Alexander and Abrami, Giuseppe + and Kleine, Lydia and Stingl, Doris}, + title = {{InterView}: Towards a Unified {VR}-Capable Interview Environment}, + year = {2026}, + howpublished = {SSRN preprint}, + doi = {10.2139/ssrn.7459949}, + url = {https://ssrn.com/abstract=7459949}, + note = {Under review} + } + ``` + +--- + +### ReEmote: Towards Emotion Representation in VR Through Va.Si.Li-Lab { #reemote } + +**Schrottenbacher, Mehler, Bernhardt, Rohe & Abrami** (2026). *Proceedings of XR Salento 2026*, Springer (accepted). + +??? note "BibTeX" + + ```bibtex + @inproceedings{Schrottenbacher:et:al:2026:a, + author = {Schrottenbacher, Patrick and Mehler, Alexander and Bernhardt, Vivienne + and Rohe, Leon and Abrami, Giuseppe}, + title = {ReEmote: Towards Emotion Representation in {VR} Through {Va.Si.Li}-Lab}, + booktitle = {Proceedings of XR Salento 2026}, + year = {2026}, + publisher = {Springer International Publishing}, + note = {accepted} + } + ``` + +--- + +### Head and Hand Movements During Turn Transitions: Data-Based Multimodal Analysis Using the Frankfurt VR Gesture–Speech Alignment Corpus (FraGA) { #fraga } + +**Lücking, Voll, Rott, Henlein & Mehler** (2025). *Proceedings of the 29th Workshop on the Semantics and Pragmatics of +Dialogue* (SemDial '25), 146–156. +[[PDF]](http://semdial.org/anthology/Z25-Luecking_semdial_3316.pdf) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Luecking:Voll:Rott:Henlein:Mehler:2025-fraga, + title = {Head and Hand Movements During Turn Transitions: Data-Based Multimodal + Analysis Using the {Frankfurt VR Gesture--Speech Alignment Corpus} + ({FraGA})}, + author = {Lücking, Andy and Voll, Felix and Rott, Daniel and Henlein, Alexander + and Mehler, Alexander}, + year = {2025}, + booktitle = {Proceedings of the 29th Workshop on The Semantics and Pragmatics + of Dialogue -- Full Papers}, + series = {SemDial'25 -- Bialogue}, + publisher = {SEMDIAL}, + url = {http://semdial.org/anthology/Z25-Luecking_semdial_3316.pdf}, + pages = {146--156}, + } + ``` + +--- + +### VR-ParlExplorer: A Hypertext System for the Collaborative Interaction in Parliamentary Debate Spaces { #vr-parlexplorer } + +**Abrami, Bundan, Manolis & Mehler** (2025). *Proceedings of the 36th ACM Conference on Hypertext and Social Media* +(HT '25), 177–183. +[[Link]](https://doi.org/10.1145/3720553.3746672) [[PDF]](https://dl.acm.org/doi/pdf/10.1145/3720553.3746672) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Abrami:et:al:2025:c, + author = {Abrami, Giuseppe and Bundan, Daniel and Manolis, Chrisowaladis + and Mehler, Alexander}, + title = {VR-ParlExplorer: A Hypertext System for the Collaborative Interaction + in Parliamentary Debate Spaces}, + year = {2025}, + isbn = {9798400715341}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3720553.3746672}, + doi = {10.1145/3720553.3746672}, + booktitle = {Proceedings of the 36th ACM Conference on Hypertext and Social Media}, + pages = {177--183}, + numpages = {7}, + location = {Chicago, USA}, + series = {HT '25}, + pdf = {https://dl.acm.org/doi/pdf/10.1145/3720553.3746672} + } + ``` + +--- + +### Geo-spatial hypertext in virtual reality: mapping and navigating global news event spaces { #news-in-time-and-space } + +**Schrottenbacher, Mehler, Berg, Hustedt, Gagel, Lüttig & Abrami** (2025). *New Review of Hypermedia and Multimedia* +31(1–2), 76–105. +[[DOI]](https://doi.org/10.1080/13614568.2024.2383601) + +??? note "BibTeX" + + ```bibtex + @article{Schrottenbacher:et:al:2025, + author = {Schrottenbacher, Patrick and Mehler, Alexander and Berg, Theresa + and Hustedt, Jasper and Gagel, Julian and Lüttig, Timo and Abrami, Giuseppe}, + title = {Geo-spatial hypertext in virtual reality: mapping and navigating + global news event spaces}, + journal = {New Review of Hypermedia and Multimedia}, + volume = {31}, + number = {1-2}, + pages = {76--105}, + year = {2025}, + publisher = {Taylor \& Francis}, + doi = {10.1080/13614568.2024.2383601}, + url = {https://doi.org/10.1080/13614568.2024.2383601}, + } + ``` + +--- + +### Viki LibraRy: Collaborative Hypertext Browsing and Navigation in Virtual Reality { #viki-library } + +**Bönisch, Mehler, Babbili, Heinrich, Stephan & Abrami** (2025). *New Review of Hypermedia and Multimedia* 31(1–2), +45–75. +[[DOI]](https://doi.org/10.1080/13614568.2024.2383581) + +??? note "BibTeX" + + ```bibtex + @article{Boenisch:et:al:2025:b, + author = {B\"{o}nisch, Kevin and Mehler, Alexander and Babbili, Shaduan + and Heinrich, Yannick and Stephan, Philipp and Abrami, Giuseppe}, + title = {{Viki LibraRy: Collaborative Hypertext Browsing and Navigation + in Virtual Reality}}, + journal = {New Review of Hypermedia and Multimedia}, + volume = {31}, + number = {1-2}, + pages = {45--75}, + year = {2025}, + publisher = {Taylor \& Francis}, + doi = {10.1080/13614568.2024.2383581}, + url = {https://doi.org/10.1080/13614568.2024.2383581}, + } + ``` --- -#### Va.Si.Li-Lab as a Collaborative Multi-User Annotation Tool in Virtual Reality and Its Potential Fields of Application - -``` bibtex -@inproceedings{Abrami:et:al:2023, - author = {Abrami, Giuseppe and Mehler, Alexander and Bagci, Mevl\"{u}t and Schrottenbacher, Patrick - and Henlein, Alexander and Spiekermann, Christian and Engel, Juliane - and Schreiber, Jakob}, - title = {Va.Si.Li-Lab as a Collaborative Multi-User Annotation Tool in - Virtual Reality and Its Potential Fields of Application}, - year = {2023}, - isbn = {9798400702327}, - publisher = {Association for Computing Machinery}, - address = {New York, NY, USA}, - url = {https://doi.org/10.1145/3603163.3609076}, - doi = {10.1145/3603163.3609076}, - booktitle = {Proceedings of the 34th ACM Conference on Hypertext and Social Media}, - articleno = {22}, - numpages = {9}, - location = {Rome, Italy}, - series = {HT '23} -} -``` -[[Link]](https://dl.acm.org/doi/10.1145/3603163.3609076) [[pdf]](https://dl.acm.org/doi/pdf/10.1145/3603163.3609076) [[RG]](https://www.researchgate.net/publication/371303212_VaSiLi-Lab_as_a_collaborative_multi-user_annotation_tool_in_virtual_reality_and_its_potential_fields_of_application) + +### Va.Si.Li-ES: VR-based Dynamic Event Processing, Environment Change and User Feedback in Va.Si.Li-Lab { #vasili-es } + +**Abrami, Wontke, Singh & Mehler** (2024). *Proceedings of the 35th ACM Conference on Hypertext and Social Media* +(HT '24), 357–368. +[[Link]](https://dl.acm.org/doi/10.1145/3648188.3675154) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Abrami:et:al:2024:b, + author = {Abrami, Giuseppe and Wontke, Dominik Alexander and Singh, Gurpreet + and Mehler, Alexander}, + title = {Va.Si.Li-ES: VR-based Dynamic Event Processing, Environment Change + and User Feedback in Va.Si.Li-Lab}, + year = {2024}, + isbn = {9798400705953}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3648188.3675154}, + doi = {10.1145/3648188.3675154}, + booktitle = {Proceedings of the 35th ACM Conference on Hypertext and Social Media}, + pages = {357--368}, + numpages = {12}, + location = {Poznan, Poland}, + series = {HT '24} + } + ``` --- +### News in Time and Space: Global Event Exploration in Virtual Reality + +**Gagel, Hustedt, Lüttig, Berg, Abrami & Mehler** (2023). *Proceedings of the 34th ACM Conference on Hypertext and +Social Media* (HT '23). +[[Link]](https://doi.org/10.1145/3603163.3609080) [[PDF]](https://dl.acm.org/doi/pdf/10.1145/3603163.3609080) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Gagel:et:al:2023, + author = {Gagel, Julian and Hustedt, Jasper and L\"{u}ttig, Timo and Berg, Theresa + and Abrami, Giuseppe and Mehler, Alexander}, + title = {News in Time and Space: Global Event Exploration in Virtual Reality}, + year = {2023}, + isbn = {9798400702327}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3603163.3609080}, + doi = {10.1145/3603163.3609080}, + booktitle = {Proceedings of the 34th ACM Conference on Hypertext and Social Media}, + articleno = {7}, + numpages = {3}, + location = {Rome, Italy}, + series = {HT '23}, + pdf = {https://dl.acm.org/doi/pdf/10.1145/3603163.3609080} + } + ``` + +--- + +### Viki LibraRy: A Virtual Reality Library for Collaborative Browsing and Navigation through Hypertext + +**Babbili, Bönisch, Heinrich, Stephan, Abrami & Mehler** (2023). *Proceedings of the 34th ACM Conference on Hypertext +and Social Media* (HT '23). +[[Link]](https://doi.org/10.1145/3603163.3609079) [[PDF]](https://dl.acm.org/doi/pdf/10.1145/3603163.3609079) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Babbili:et:al:2023, + author = {Babbili, Shaduan and B\"{o}nisch, Kevin and Heinrich, Yannick + and Stephan, Philipp and Abrami, Giuseppe and Mehler, Alexander}, + title = {Viki LibraRy: A Virtual Reality Library for Collaborative Browsing + and Navigation through Hypertext}, + year = {2023}, + isbn = {9798400702327}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3603163.3609079}, + doi = {10.1145/3603163.3609079}, + booktitle = {Proceedings of the 34th ACM Conference on Hypertext and Social Media}, + articleno = {6}, + numpages = {3}, + location = {Rome, Italy}, + series = {HT '23}, + pdf = {https://dl.acm.org/doi/pdf/10.1145/3603163.3609079} + } + ``` + +--- + +### Semantic Scene Builder: Towards a Context Sensitive Text-to-3D Scene Framework { #semantic-scene-builder } + +**Henlein, Kett, Baumartz, Abrami, Mehler et al.** (2023). *Digital Human Modeling and Applications in Health, Safety, +Ergonomics and Risk Management* (HCII 2023), Springer, 461–479. +[[DOI]](https://doi.org/10.1007/978-3-031-35748-0_32) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Henlein:et:al:2023b, + author = {Henlein, Alexander and Kett, Attila and Baumartz, Daniel and Abrami, Giuseppe + and Mehler, Alexander and Bastian, Johannes and Blecher, Yannic and Budgenhagen, David + and Christof, Roman and Ewald, Tim-Oliver and Fauerbach, Tim and Masny, Patrick + and Mende, Julian and Schn{\"u}re, Paul and Viel, Marc}, + editor = {Duffy, Vincent G.}, + title = {Semantic Scene Builder: Towards a Context Sensitive Text-to-3D Scene Framework}, + booktitle = {Digital Human Modeling and Applications in Health, Safety, Ergonomics + and Risk Management}, + year = {2023}, + publisher = {Springer Nature Switzerland}, + address = {Cham}, + pages = {461--479}, + isbn = {978-3-031-35748-0}, + doi = {10.1007/978-3-031-35748-0_32}, + } + ``` ## Presentations / Posters -#### Towards grounding multimodal semantics in interaction data with Va.Si.Li-Lab - -``` bibtex -@inproceedings{Henlein:et:al:2023c, - title = {Towards grounding multimodal semantics in interaction data with Va.Si.Li-Lab}, - author = {Henlein, Alexander and Lücking, Andy and Bagci, Mevlüt and Mehler, Alexander}, - booktitle = {Proceedings of the 8th Conference on Gesture and Speech in Interaction (GESPIN)}, - location = {Nijmegen, Netherlands}, - year = {2023}, - keywords = {vasililab}, - pdf = {https://www.gespin2023.nl/documents/talks_and_posters/GeSpIn_2023_papers/GeSpIn_2023_paper_1692.pdf} -} -``` -[[pdf]](https://www.gespin2023.nl/documents/talks_and_posters/GeSpIn_2023_papers/GeSpIn_2023_paper_1692.pdf) - ---- \ No newline at end of file +### Virtually Restricting Modalities in Interactions: Va.Si.Li-Lab for Experimental Multimodal Research + +**Henlein, Lücking & Mehler** (2024). *2nd International Symposium on Multimodal Communication* (MMSYM 2024), 96–97. +[[PDF]](http://mmsym.org/wp-content/uploads/2024/09/BookOfAbstractsMMSYM2024-3.pdf) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Henlein:Luecking:Mehler:2024, + title = {Virtually Restricting Modalities in Interactions: Va.Si.Li-Lab + for Experimental Multimodal Research}, + author = {Henlein, Alexander and L{\"u}cking, Andy and Mehler, Alexander}, + booktitle = {Proceedings of the 2nd International Symposium on Multimodal Communication + (MMSYM 2024), Frankfurt, 25-27 September 2024}, + pages = {96--97}, + year = {2024}, + pdf = {http://mmsym.org/wp-content/uploads/2024/09/BookOfAbstractsMMSYM2024-3.pdf}, + } + ``` + +--- + +### Towards grounding multimodal semantics in interaction data with Va.Si.Li-Lab + +**Henlein, Lücking, Bagci & Mehler** (2023). *Proceedings of the 8th Conference on Gesture and Speech in Interaction* +(GESPIN 2023), Nijmegen. +[[PDF]](https://www.gespin2023.nl/documents/talks_and_posters/GeSpIn_2023_papers/GeSpIn_2023_paper_1692.pdf) + +??? note "BibTeX" + + ```bibtex + @inproceedings{Henlein:et:al:2023c, + title = {Towards grounding multimodal semantics in interaction data with Va.Si.Li-Lab}, + author = {Henlein, Alexander and Lücking, Andy and Bagci, Mevlüt and Mehler, Alexander}, + booktitle = {Proceedings of the 8th Conference on Gesture and Speech in Interaction (GESPIN)}, + location = {Nijmegen, Netherlands}, + year = {2023}, + pdf = {https://www.gespin2023.nl/documents/talks_and_posters/GeSpIn_2023_papers/GeSpIn_2023_paper_1692.pdf} + } + ``` diff --git a/Documentation/docs/scenarios/DGS.md b/Documentation/docs/scenarios/DGS.md deleted file mode 100644 index bf965699..00000000 --- a/Documentation/docs/scenarios/DGS.md +++ /dev/null @@ -1,2 +0,0 @@ -# DGS -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/scenarios/ICIDS.md b/Documentation/docs/scenarios/ICIDS.md deleted file mode 100644 index 71f430eb..00000000 --- a/Documentation/docs/scenarios/ICIDS.md +++ /dev/null @@ -1,2 +0,0 @@ -# ICIDS -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/scenarios/Meetingpoint.md b/Documentation/docs/scenarios/Meetingpoint.md deleted file mode 100644 index 2c48d0eb..00000000 --- a/Documentation/docs/scenarios/Meetingpoint.md +++ /dev/null @@ -1,2 +0,0 @@ -# Meetingpoint -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/scenarios/OrganisationDistribution.md b/Documentation/docs/scenarios/OrganisationDistribution.md deleted file mode 100644 index 4640f583..00000000 --- a/Documentation/docs/scenarios/OrganisationDistribution.md +++ /dev/null @@ -1,2 +0,0 @@ -# Organisation Distribution -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/scenarios/OrganisationEducation.md b/Documentation/docs/scenarios/OrganisationEducation.md deleted file mode 100644 index 2c805697..00000000 --- a/Documentation/docs/scenarios/OrganisationEducation.md +++ /dev/null @@ -1,2 +0,0 @@ -# Organisation Education -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/scenarios/PraktikumScenario1.md b/Documentation/docs/scenarios/PraktikumScenario1.md deleted file mode 100644 index 67914a6d..00000000 --- a/Documentation/docs/scenarios/PraktikumScenario1.md +++ /dev/null @@ -1,35 +0,0 @@ -# Praktikum Scenario 1 - -## Description -Three people take part in the experiment. There are two runs. -Run 1: One person sees the flat furnished with abstract objects. The other two -The other two people have to use the descriptions to get the right objects from the storage and place them in the -and place them in the right place. The participants have random restrictions (sight, hearing, nothing). -After the first round, the participants can exchange ideas and tips with the restrictions and roles. -restrictions and roles. -Round 2: The roles and restrictions are then redistributed and the experiment is repeated. - - -## Parameter -* Number of participants: 3 -* Duration: 2h - - -## Rounds - -### Round 1 -* One person sees the flat furnished. -* The rooms are numbered on the doors (the large room in the centre has the number 8). -* The task is to describe the objects in the rooms in order so that the other people can get the right objects from the storage room and place them correctly. -* The placement does not have to be perfect. -* The round lasts 25min. Then person 3 should finish the round. (Stop the time!) - -### Round 2 -* All participants are allowed to exchange ideas and report on their experiences in the role and any limitations. -* The aim is to make it easier for the other participants to start the next round. -* When everyone has exchanged ideas, person 3 can start the next round. - -### Round 3 -* See round 1. -* The roles, restrictions and objects in the flat are redistributed randomly. -* At the end, simply close applications again and remove glasses. diff --git a/Documentation/docs/scenarios/PraktikumScenario2.md b/Documentation/docs/scenarios/PraktikumScenario2.md deleted file mode 100644 index 3ce74b7f..00000000 --- a/Documentation/docs/scenarios/PraktikumScenario2.md +++ /dev/null @@ -1,55 +0,0 @@ -# Praktikum Scenario 2 - Short Films - -## Description -In 4 rounds, the participants are each shown a different film at the same time. -After both people have seen their film (after each film), -they meet again in the living room (are automatically teleported) and are asked to give a summary of these films. - - -## Parameter -* Number of participants: 2 -* Duration: 1h - - -## Rounds - -### Round 1 -- As soon as everyone has clicked on "Bereit" and the round starts, they both watch their film. -- As soon as person 1 has finished the film, the level window appears and the level can be completed. -- As soon as person 2 has also finished the film, they can click on \enquote{Ready} and person 1 can start the next round. -- If person 2 finishes the film earlier, they simply wait until the \enquote{Ready} field appears in the level window. - -### Round 2 -- Both people face each other in VR and put down their controllers. -- First, person 1 narrates their film. -- After the narration, questions are welcome. -- At the end, person 1 should rate the film on a scale from 1 (bad) to 5 (good). -- Then person 2 narrates their film. -- After the narration, questions are welcome. -- Finally, person 2 should rate the film on a scale from 1 (bad) to 5 (good). -- (The ratings should later be used as a trigger to separate the two narratives) -- As soon as both people have retold their film, person 1 can end the level again (record controller for this). - -### Round 3 -- See round 1. - -### Round 4 -- See round 2. - -### Round 5-8 -- See round 1 - 4. - -## Movies -### Group 1 -- [For The Birds](https://youtu.be/nYTrIcn4rjg) -- [Snack Attack](https://youtu.be/38y_1EWIE9I) -### Group 2 -- [In the Green](https://youtu.be/nYTrIcn4rjg) -- [Mars](https://youtu.be/BmocltqxRak) -### Group 3 -- [The Mug](https://youtu.be/1y3h0B2b-HA) -- [The Black Hole](https://youtu.be/P5_Msrdg3Hk) -### Group 4 -- [Dji. Death Sails](https://www.youtube.com/OlOdyvZWtDQ) -- [The Choice](https://www.youtube.com/_HEnohs6yYw) - diff --git a/Documentation/docs/scenarios/PraktikumScenario3.md b/Documentation/docs/scenarios/PraktikumScenario3.md deleted file mode 100644 index ac67646e..00000000 --- a/Documentation/docs/scenarios/PraktikumScenario3.md +++ /dev/null @@ -1,29 +0,0 @@ -# Praktikum Scenario 3 - -## Description -Person 1 is driven through the fictitious [SaGA](https://pub.uni-bielefeld.de/record/2001935) city. -Person 1 then explains the route to person 2. -Person 2 then has to find their way through the SaGA city. - -## Parameter -* Number of participants: 3 -* Duration: 1h - - -## Rounds - -### Round 1 -- Person 1 is teleported to the starting point. As soon as the round begins, the city tour starts. -- The city tour itself lasts 7 minutes. -- Person 2 can be bored in the living room during this time. - -### Round 2 -- Both people face each other in VR and put away their controllers. -- Person 1 now has the task of explaining the route to person 2. -- At the end, both players can pick up their controllers again. -- Player 1 finishes the level and both get ready. - -### Round 3 -- Person 1 starts the round and can then twiddle their thumbs. -- Person 2 is teleported to the starting point and has the task of finding the way based on the previous directions. -- As soon as person 2 is hopelessly lost, the round can be cancelled. diff --git a/Documentation/docs/scenarios/SchoolDistribution.md b/Documentation/docs/scenarios/SchoolDistribution.md deleted file mode 100644 index 5899df23..00000000 --- a/Documentation/docs/scenarios/SchoolDistribution.md +++ /dev/null @@ -1,2 +0,0 @@ -# School Distribution -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/scenarios/SchoolHetero.md b/Documentation/docs/scenarios/SchoolHetero.md deleted file mode 100644 index a3e24da9..00000000 --- a/Documentation/docs/scenarios/SchoolHetero.md +++ /dev/null @@ -1,2 +0,0 @@ -# School Hetero -This documentation is work in progress. \ No newline at end of file diff --git a/Documentation/docs/scenarios/index.md b/Documentation/docs/scenarios/index.md new file mode 100644 index 00000000..7a90af12 --- /dev/null +++ b/Documentation/docs/scenarios/index.md @@ -0,0 +1,11 @@ +# Scenarios + +A scenario is a scene plus its [scene definition](../getting_started/adding_scenes.md) (roles, levels, spawn points). +Scenario scenes made for studies are kept in the projects that run them; the public repository contains the +[sample scenes](../getting_started/special_scenes.md): + +| Scene | Purpose | +|---|---| +| `Example` | Minimal scenario to start a new scene from | +| `InterViewStart`, `InterView`, `InterViewReturn` | [InterView](../systems.md#interview): token entry, the interview room (seat, mirror, in-world browser, WebRTC connection to the interviewer) and the end screen | +| `Tutorial` | Learn the controls | diff --git a/Documentation/docs/server/server_structure.md b/Documentation/docs/server/server_structure.md index 9091d608..f84335de 100644 --- a/Documentation/docs/server/server_structure.md +++ b/Documentation/docs/server/server_structure.md @@ -7,7 +7,12 @@ depending on the scenario. |---|---|---| | [Ubiq Server](ubiq_server.md) | Room and session state - avatars, transforms, ownership | required | | [Database API](python_data_logger.md) | Scene definitions and the logging API, backed by [MongoDB](mongodb.md) | required | -| Chatbot, Speech, VR Agent | Conversational agents, speech-to-text and text-to-speech | optional | +| [VR Agent](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/vr_agent/) (`vr-agent/`) | LLM agents for the [Bots](../modules/bots.md) | optional | +| [TTS/STT](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/tts_stt/) (`tts-stt/`) | Speech-to-text and text-to-speech for the agents | optional | +| [Speech](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/speech/) (`speech2text/`) | Whisper speech-to-text for voice input | optional | +| [Chatbot](https://texttechnologylab.github.io/Va.Si.Li-Lab-backend/services/chatbot/) (`chatbot/`) | Older voice chatbot host | optional | + +Which client setting points to which service is listed in [Endpoints Configuration](../modules/endpoints.md). All of them live in the [Va.Si.Li-Lab-backend](https://github.com/texttechnologylab/Va.Si.Li-Lab-backend) diff --git a/Documentation/docs/server/ubiq_server.md b/Documentation/docs/server/ubiq_server.md index cd9d06a4..8064f9b4 100644 --- a/Documentation/docs/server/ubiq_server.md +++ b/Documentation/docs/server/ubiq_server.md @@ -43,8 +43,12 @@ docker run -d --restart unless-stopped -p 8009:8009 -p 8011:8011 ubiq ## Unity -`SocialNetworkScene > RoomClient` -### Docker Compose +The client reads the server address from *Ubiq Server* (a Ubiq `ConnectionDefinition`) in +`Assets/Resources/VasiliEndpoints.asset`. Create the file with *Tools > Va.Si.Li > Create Endpoints Config*. The +`VasiliServerConnection` component on the `RoomClient` of the `Social Network Scene` prefab applies it at start-up. If +the file is missing, the client falls back to `127.0.0.1:8009` and logs a warning. + +## Docker Compose The Ubiq server is also part of the combined [InterView deployment stack](https://texttechnologylab.github.io/InterView/services/ubiq/), diff --git a/Documentation/docs/students/documentation.md b/Documentation/docs/students/documentation.md index c01eac8e..9bac65bc 100644 --- a/Documentation/docs/students/documentation.md +++ b/Documentation/docs/students/documentation.md @@ -5,22 +5,30 @@ It is written in [Markdown](https://www.markdownguide.org/basic-syntax/) and can ## Testing locally -### Requirements +With [uv](https://docs.astral.sh/uv/getting-started/installation/) installed, no further setup is needed. From the +repository root: + ```bash -conda create -n mkdocs python=3.12 -conda activate mkdocs -pip install mkdocs-material -pip install mkdocs-git-revision-date-localized-plugin mkdocs-git-committers-plugin-2 mkdocs-git-authors-plugin mkdocs-glightbox +cd Documentation +uvx --with mkdocs-material --with mkdocs-git-revision-date-localized-plugin --with mkdocs-glightbox mkdocs serve ``` -### Run +Then open . Pages reload when you save. + +Before opening a pull request, check that the docs build without warnings, as the CI does: ```bash -cd Documentation -mkdocs serve +uvx --with mkdocs-material --with mkdocs-git-revision-date-localized-plugin --with mkdocs-glightbox mkdocs build --strict ``` +Without uv, install the same packages with pip (`pip install mkdocs-material mkdocs-git-revision-date-localized-plugin +mkdocs-glightbox`) and run `mkdocs serve`. + ## Adding a new page To add a new page, create a new markdown file in the `Documentation/docs` folder. Then insert the file path in the `mkdocs.yml` file under the `nav` section. + +## Publishing +The docs are built and published to GitHub Pages on every push to `main` and on every pull request, so a pull +request shows up on the live site before it is merged. diff --git a/Documentation/docs/systems.md b/Documentation/docs/systems.md new file mode 100644 index 00000000..64df28c7 --- /dev/null +++ b/Documentation/docs/systems.md @@ -0,0 +1,65 @@ +# Systems using Va.Si.Li-Lab + +A list of Systems built on Va.Si.Li-Lab, as modules of this repository or as separate applications. Papers and BibTeX are listed +under [Publications](publications.md#systems-using-vasili-lab). + +| System | What it does | Paper | Part of | +|---|---|---|---| +| [InterView](#interview) | VR interviews: a participant in VR, the interviewer in a web browser | [Preprint](https://doi.org/10.2139/ssrn.7459949) | [FACES](projects/faces.md) | +| [ReEmote](#reemote) | Emotion representation for avatars and their environment | [XR Salento 2026](publications.md#reemote) (accepted) | [FACES](projects/faces.md) | +| [Va.Si.Li-ES](#vasili-es) | Event system: triggers, environment and weather changes, voting and feedback | [HT '24](https://doi.org/10.1145/3648188.3675154) | This repository (`Assets/Lab/EventSystem`) | +| [VR-ParlExplorer](#vr-parlexplorer) | Plenary debates in VR with chatbot members of parliament | [HT '25](https://doi.org/10.1145/3720553.3746672) | | +| [News in Time and Space](#news-in-time-and-space) | Geospatial hypertext: global news events on a virtual globe | [NRHM 2025](https://doi.org/10.1080/13614568.2024.2383601) | | +| [Viki LibraRy](#viki-library) | A collaborative VR library for browsing and reading hypertext | [NRHM 2025](https://doi.org/10.1080/13614568.2024.2383581) | | +| [Semantic Scene Builder](#semantic-scene-builder) | Text-to-3D scene generation, edited in VR | [HCII 2023](https://doi.org/10.1007/978-3-031-35748-0_32) | [ViCom](projects/vicom.md) | +| [FraGA](#fraga) | Frankfurt VR Gesture–Speech Alignment corpus, recorded in Va.Si.Li-Lab | [SemDial 2025](http://semdial.org/anthology/Z25-Luecking_semdial_3316.pdf) | [ViCom](projects/vicom.md) | + +## InterView + +A platform for VR survey interviews. The participant wears a headset and sits across from the interviewer's avatar; +the interviewer works from an ordinary browser. Speech, gaze, posture, hand movement, facial expression and the answers +are captured on a shared clock. InterView adds a WebRTC media layer (Janus), a browser-based interviewer console and a +token-driven session model to Va.Si.Li-Lab. + +- Platform documentation: [texttechnologylab.github.io/InterView](https://texttechnologylab.github.io/InterView/) +- VR client setup: [InterView VR Client](getting_started/interview_client.md) +- Paper: [InterView: Towards a Unified VR-Capable Interview Environment](publications.md#interview) + +## ReEmote + +Extends the face and hand tracking of Va.Si.Li-Lab with schema-based emotion mappings that affect both avatars and +their environments, so users and virtual agents share one emotional expression space. +Paper: [ReEmote](publications.md#reemote). + +## Va.Si.Li-ES + +The event system of Va.Si.Li-Lab. Events are created and serialised at run time and change situational and +environmental parameters (weather, objects, triggers); users are informed and can take part through voting and +feedback. It ships as its own package, see `Assets/Lab/EventSystem/README.md`. +Paper: [Va.Si.Li-ES](publications.md#vasili-es). + +## VR-ParlExplorer + +A plugin for Va.Si.Li-Lab that virtualises plenary debates: users interact with virtual members of parliament through +LLM-based chatbots. Paper: [VR-ParlExplorer](publications.md#vr-parlexplorer). + +## News in Time and Space + +News in Time and Space (NiTS) organises news articles (currently based on GDELT) as a geospatial hypertext on a virtual +globe, where users visualise, filter and explore global events and trends. +Paper: [Geo-spatial hypertext in virtual reality](publications.md#news-in-time-and-space). + +## Viki LibraRy + +A dynamically built library in VR that visualises hypertext systems, with an emphasis on collaborative reading and +spatial recall. Paper: [Viki LibraRy](publications.md#viki-library). + +## Semantic Scene Builder + +A VR-based text-to-3D framework: scenes are generated from annotated text descriptions and corrected by grabbing and +moving objects instead of editing annotations. Paper: [Semantic Scene Builder](publications.md#semantic-scene-builder). + +## FraGA + +The Frankfurt VR Gesture–Speech Alignment corpus, used for multimodal analyses of head and hand movements in dialogue. +Paper: [FraGA](publications.md#fraga). diff --git a/Documentation/mkdocs.yml b/Documentation/mkdocs.yml index 5ded50a3..a55087ba 100644 --- a/Documentation/mkdocs.yml +++ b/Documentation/mkdocs.yml @@ -22,23 +22,19 @@ nav: - InterView VR Client: ./getting_started/interview_client.md + - Modules: + - Endpoints Configuration: ./modules/endpoints.md + - Event System: ./modules/eventsystem.md + - Bots: ./modules/bots.md + - Logging: ./modules/logging.md + - Other Functions: - Replay Experiment: ./other/replay.md - - Bots: ./other/bots.md - - Experiment Visualization Audio: ./other/experiment_visualizer_audio.md + - Experiment Visualization: ./other/experiment_visualization.md - Scenarios: - - ICIDS: ./scenarios/ICIDS.md - - PraktikumScenario1: ./scenarios/PraktikumScenario1.md - - PraktikumScenario2: ./scenarios/PraktikumScenario2.md - - PraktikumScenario3: ./scenarios/PraktikumScenario3.md - - DGS: ./scenarios/DGS.md - - Meetingpoint: ./scenarios/Meetingpoint.md - - OrganisationDistribution: ./scenarios/OrganisationDistribution.md - - OrganisationEducation: ./scenarios/OrganisationEducation.md - - SchoolDistribution: ./scenarios/SchoolDistribution.md - - SchoolHetero: ./scenarios/SchoolHetero.md + - Overview: ./scenarios/index.md - Server Structure: - General: ./server/server_structure.md @@ -50,10 +46,13 @@ nav: - Documentation: ./students/documentation.md - Projects: - - Digitell: ./projects/digitell.md + - Overview: ./projects/index.md + - DigiTeLL: ./projects/digitell.md - ViCom: ./projects/vicom.md - New Data Spaces (FACES): ./projects/faces.md + - Systems: systems.md + - Publications: publications.md @@ -142,6 +141,7 @@ markdown_extensions: - pymdownx.superfences - admonition + - pymdownx.details - attr_list - md_in_html diff --git a/Packages/manifest.json b/Packages/manifest.json index 108e4025..2b8bace2 100644 --- a/Packages/manifest.json +++ b/Packages/manifest.json @@ -25,7 +25,7 @@ "com.unity.timeline": "1.8.13", "com.unity.ugui": "2.0.0", "com.unity.vectorgraphics": "3.0.0-preview.7", - "com.unity.webrtc": "3.0.0", + "com.unity.webrtc-ubiq-fork": "https://github.com/UCL-VR/unity-webrtc-ubiq-fork.git#v3.0.1-pre.7", "com.unity.xr.hands": "1.7.3", "com.unity.xr.interaction.toolkit": "3.3.2", "com.unity.xr.management": "4.6.1", diff --git a/Packages/packages-lock.json b/Packages/packages-lock.json index 57e54183..ebf1047d 100644 --- a/Packages/packages-lock.json +++ b/Packages/packages-lock.json @@ -403,16 +403,16 @@ }, "url": "https://packages.unity.com" }, - "com.unity.webrtc": { - "version": "3.0.0", + "com.unity.webrtc-ubiq-fork": { + "version": "https://github.com/UCL-VR/unity-webrtc-ubiq-fork.git#v3.0.1-pre.7", "depth": 0, - "source": "registry", + "source": "git", "dependencies": { - "com.unity.modules.audio": "1.0.0", + "com.unity.modules.jsonserialize": "1.0.0", "com.unity.editorcoroutines": "1.0.0", - "com.unity.modules.jsonserialize": "1.0.0" + "com.unity.modules.audio": "1.0.0" }, - "url": "https://packages.unity.com" + "hash": "1724f81b5c39bb4f44bf550de64b9f1fa318ccc8" }, "com.unity.xr.core-utils": { "version": "2.6.0", diff --git a/ProjectSettings/EditorBuildSettings.asset b/ProjectSettings/EditorBuildSettings.asset index c5552602..835cdd0c 100644 --- a/ProjectSettings/EditorBuildSettings.asset +++ b/ProjectSettings/EditorBuildSettings.asset @@ -5,9 +5,6 @@ EditorBuildSettings: m_ObjectHideFlags: 0 serializedVersion: 2 m_Scenes: - - enabled: 1 - path: Assets/LabPrivate/Scenes/FACES_.unity - guid: 912de1c6a9b21a044affe21425816298 - enabled: 1 path: Assets/Lab/Core/Samples/Scenes/MetaStart.unity guid: e6d9483601d135240bca91bb89b1b894 @@ -15,7 +12,7 @@ EditorBuildSettings: path: Assets/Lab/Core/Samples/Scenes/MetaReturn.unity guid: a20c8cfd9870a3942b37f0eaa4077803 m_configObjects: - Unity.XR.Oculus.Settings: {fileID: 11400000, guid: 2acd84852d7486c4485f42afca1ff615, type: 2} + Unity.XR.Oculus.Settings: {fileID: 11400000, guid: fb897279f8412534bbb2ec6af5835e6e, type: 2} com.unity.xr.management.loader_settings: {fileID: 11400000, guid: e5d9c8f67f1518c429bb8eca89e53dc9, type: 2} com.unity.xr.openxr.settings4: {fileID: 11400000, guid: 930343fcbf66e2d4cba10cfe8640789e, type: 2} m_UseUCBPForAssetBundles: 0 diff --git a/README.md b/README.md index 5f010ac5..8be10ddc 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,13 @@ -[![Paper_HCII](http://img.shields.io/badge/paper-HCII--2023-B31B1B.svg)](https://doi.org/10.1007/978-3-031-35741-1_39) -[![Paper_HT](http://img.shields.io/badge/paper-HT--2023-F31B1B.svg)](https://doi.org/10.1145/3603163.3609076) -[![Conference](http://img.shields.io/badge/conference-HCII--2023-4b44ce.svg)](https://2023.hci.international/) -[![version](https://img.shields.io/github/license/texttechnologylab/Va.Si.Li-Lab)]() -![GitHub release (with filter)](https://img.shields.io/github/v/release/texttechnologylab/Va.Si.Li-Lab) -[![Documentation](https://img.shields.io/badge/Material_for_MkDocs-526CFE?style=for-the-badge&logo=MaterialForMkDocs&logoColor=white)](https://texttechnologylab.github.io/Va.Si.Li-Lab/#citation) +

+ Va.Si.Li-Lab +

+ +[![Paper_HCII](https://img.shields.io/badge/paper-HCII--2023-B31B1B.svg)](https://doi.org/10.1007/978-3-031-35741-1_39) +[![Paper_HT](https://img.shields.io/badge/paper-HT--2023-F31B1B.svg)](https://doi.org/10.1145/3603163.3609076) +[![Paper_TKL](https://img.shields.io/badge/paper-TKL--2025-4b44ce.svg)](https://doi.org/10.1007/s10758-025-09837-7) +[![License](https://img.shields.io/github/license/texttechnologylab/Va.Si.Li-Lab)](LICENSE) +[![GitHub release](https://img.shields.io/github/v/release/texttechnologylab/Va.Si.Li-Lab)](https://github.com/texttechnologylab/Va.Si.Li-Lab/releases) +[![Documentation](https://img.shields.io/badge/Documentation-526CFE?logo=MaterialForMkDocs&logoColor=white)](https://texttechnologylab.github.io/Va.Si.Li-Lab/) # Va.Si.Li-Lab a **V**R-L**a**b for **Si**mulation-based **L**earn**i**ng @@ -12,20 +16,68 @@ a **V**R-L**a**b for **Si**mulation-based **L**earn**i**ng Va.Si.Li-Lab was established as part of the project "Digital Teaching and Learning Lab" (DigiTeLL) at the Goethe University Frankfurt. It has since evolved into a highly capable and flexible VR framework for enabling simultaneous, multi-user engagements. +# Projects +| Project | Va.Si.Li-Lab is used for | +|---|---| +| [DigiTeLL](https://texttechnologylab.github.io/Va.Si.Li-Lab/projects/digitell/) (Goethe University Frankfurt) | Origin of Va.Si.Li-Lab: simulation-based learning in VR for teaching | +| [ViCom / GeMDiS](https://texttechnologylab.github.io/Va.Si.Li-Lab/projects/vicom/) (DFG SPP 2392) | Gestures in dialogue: VR-based multimodal data collection | +| [New Data Spaces / FACES](https://texttechnologylab.github.io/Va.Si.Li-Lab/projects/faces/) (DFG SPP 2431) | Survey interviews in VR with [InterView](https://texttechnologylab.github.io/InterView/), see the [FACES studies](https://texttechnologylab.github.io/FACES/studies/) | + +# Systems using Va.Si.Li-Lab +* [InterView](https://texttechnologylab.github.io/InterView/): VR survey interviews, with the interviewer in a web browser ([paper](https://doi.org/10.2139/ssrn.7459949)) +* ReEmote: emotion representation for avatars and their environment ([paper](https://texttechnologylab.github.io/Va.Si.Li-Lab/publications/#reemote), XR Salento 2026, accepted) +* Va.Si.Li-ES: event system for environment changes, voting and feedback, `Assets/Lab/EventSystem` ([paper](https://doi.org/10.1145/3648188.3675154)) +* VR-ParlExplorer: plenary debates in VR with chatbot members of parliament ([paper](https://doi.org/10.1145/3720553.3746672)) +* News in Time and Space: global news events as geospatial hypertext on a virtual globe ([paper](https://doi.org/10.1080/13614568.2024.2383601)) +* Viki LibraRy: a collaborative VR library for browsing hypertext ([paper](https://doi.org/10.1080/13614568.2024.2383581)) +* Semantic Scene Builder: text-to-3D scene generation, edited in VR ([paper](https://doi.org/10.1007/978-3-031-35748-0_32)) +* FraGA: the Frankfurt VR Gesture–Speech Alignment corpus ([paper](http://semdial.org/anthology/Z25-Luecking_semdial_3316.pdf)) + +Details: [Systems](https://texttechnologylab.github.io/Va.Si.Li-Lab/systems/) · [Publications](https://texttechnologylab.github.io/Va.Si.Li-Lab/publications/) -# Va.Si.Li-Lab - Team -* Prof. Dr. Alexander Mehler (Leader) -* Giuseppe Abrami -* Mevlüt Bagci -* Dr. Alexander Henlein -* Patrick Schrottenbacher +# Core Team +Current members of the [Text Technology Lab](https://www.texttechnologylab.org/team/) working on Va.Si.Li-Lab: + +* Prof. Dr. Alexander Mehler (Leader) [@amehler](https://github.com/amehler) +* Giuseppe Abrami [@abrami](https://github.com/abrami) +* Mevlüt Bagci [@mevbagci](https://github.com/mevbagci) +* Dr. Alexander Henlein [@henlein](https://github.com/henlein) +* Patrick Schrottenbacher [@TheBv](https://github.com/TheBv) # Documentation https://texttechnologylab.github.io/Va.Si.Li-Lab/ +To use Va.Si.Li-Lab as a Unity package, see [Assets/Lab/Core/README.md](Assets/Lab/Core/README.md). # Cite -If you want to use the project or the corpus, please quote this as follows: [BibTeX](https://texttechnologylab.github.io/Va.Si.Li-Lab/publications/) +If you want to use the project, please cite: + +```bibtex +@inproceedings{Mehler:et:al:2023:a, + author = {Mehler, Alexander and Bagci, Mevl{\"u}t and Henlein, Alexander + and Abrami, Giuseppe and Spiekermann, Christian and Schrottenbacher, Patrick + and Konca, Maxim and L{\"u}cking, Andy and Engel, Juliane and Quintino, Marc + and Schreiber, Jakob and Saukel, Kevin and Zlatkin-Troitschanskaia, Olga}, + booktitle = {Digital Human Modeling and Applications in Health, Safety, Ergonomics + and Risk Management}, + editor = {Duffy, Vincent G.}, + address = {Cham}, + pages = {539--565}, + publisher = {Springer Nature Switzerland}, + title = {A Multimodal Data Model for Simulation-Based Learning with Va.Si.Li-Lab}, + year = {2023}, + doi = {10.1007/978-3-031-35741-1_39} +} +``` + +All publications with BibTeX: [Publications](https://texttechnologylab.github.io/Va.Si.Li-Lab/publications/) +# Contributors +Thank you to everyone who has contributed to Va.Si.Li-Lab! The list below is generated from the +[GitHub contributors](https://github.com/texttechnologylab/Va.Si.Li-Lab/graphs/contributors) of this repository and +needs no maintenance. + + Contributors + diff --git a/Va.Si.Li-Lab-public.slnx b/Va.Si.Li-Lab-public.slnx new file mode 100644 index 00000000..c4e1e41c --- /dev/null +++ b/Va.Si.Li-Lab-public.slnx @@ -0,0 +1,254 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +