Skip to content

Commit 3d8e9a7

Browse files
vis-primevis-prime
authored
feat: add Fisheye (#96)
* feat: add Fisheye class and story to showcase functionality
1 parent 18ebcf7 commit 3d8e9a7

File tree

4 files changed

+386
-0
lines changed

4 files changed

+386
-0
lines changed

‎.storybook/stories/Fisheye.stories.ts‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import * as THREE from 'three'
2+
import { Setup } from '../Setup'
3+
import { Meta } from '@storybook/html'
4+
import { EXRLoader } from 'three/examples/jsm/loaders/EXRLoader.js'
5+
import { GroundedSkybox } from 'three/examples/jsm/objects/GroundedSkybox.js'
6+
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js'
7+
import { GUI } from 'lil-gui'
8+
import { Fisheye } from '../../src/core/Fisheye.ts'
9+
10+
export default {
11+
title: 'Portals/Fisheye',
12+
} as Meta // TODO: this should be `satisfies Meta` but commit hooks lag behind TS
13+
let gui: GUI
14+
15+
let cylinderMeshes: THREE.Mesh[] = []
16+
let fisheye: Fisheye
17+
18+
const params = {
19+
fisheyeEnabled: true,
20+
zoom: 0,
21+
resolution: 896,
22+
}
23+
24+
export const FisheyeStory = async () => {
25+
gui = new GUI({ title: 'Fisheye Story', closeFolders: true })
26+
const { renderer, scene, camera } = Setup()
27+
renderer.shadowMap.enabled = true
28+
camera.position.set(0, 1, 2)
29+
const controls = new OrbitControls(camera, renderer.domElement)
30+
controls.target.set(0, 1, 0)
31+
controls.update()
32+
33+
const floor = new THREE.Mesh(
34+
new THREE.PlaneGeometry(60, 60).rotateX(-Math.PI / 2),
35+
new THREE.ShadowMaterial({ opacity: 0.3 })
36+
)
37+
floor.receiveShadow = true
38+
scene.add(floor)
39+
40+
const dirLight = new THREE.DirectionalLight(0xabcdef, 5)
41+
dirLight.position.set(15, 15, 15)
42+
dirLight.castShadow = true
43+
dirLight.shadow.mapSize.width = 1024
44+
dirLight.shadow.mapSize.height = 1024
45+
dirLight.shadow.camera.top = 15
46+
dirLight.shadow.camera.bottom = -15
47+
dirLight.shadow.camera.left = -15
48+
dirLight.shadow.camera.right = 15
49+
scene.add(dirLight)
50+
51+
fisheye = new Fisheye({
52+
resolution: params.resolution,
53+
zoom: params.zoom,
54+
})
55+
56+
addFisheyeGUI(fisheye)
57+
58+
setupEnvironment(scene)
59+
addMeshes(scene)
60+
setupRaycaster(renderer, camera, cylinderMeshes)
61+
62+
const onWindowResize = () => {
63+
// renderer and camera handled by Setup.ts
64+
fisheye.onResize(window.innerWidth, window.innerHeight)
65+
}
66+
window.addEventListener('resize', onWindowResize)
67+
onWindowResize()
68+
69+
renderer.setAnimationLoop(() => {
70+
controls.update()
71+
if (params.fisheyeEnabled) {
72+
fisheye.render(renderer, scene, camera)
73+
} else {
74+
renderer.render(scene, camera)
75+
}
76+
})
77+
}
78+
79+
const addMeshes = (scene: THREE.Scene) => {
80+
const geo = new THREE.CylinderGeometry(0.25, 0.25, 1, 32).translate(0, 0.5, 0)
81+
82+
const count = 20
83+
for (let i = 0; i < count; i++) {
84+
const mesh = new THREE.Mesh(
85+
geo,
86+
new THREE.MeshStandardMaterial({ color: new THREE.Color().setHSL(i / count, 1, 0.5), roughness: 0.5 })
87+
)
88+
mesh.scale.y = THREE.MathUtils.randFloat(1, 5)
89+
mesh.position.set(THREE.MathUtils.randFloatSpread(15), 0, THREE.MathUtils.randFloatSpread(15))
90+
mesh.castShadow = true
91+
mesh.receiveShadow = true
92+
scene.add(mesh)
93+
cylinderMeshes.push(mesh) // for raycasting
94+
}
95+
}
96+
97+
/**
98+
* Add scene.environment and groundProjected skybox
99+
*/
100+
const setupEnvironment = (scene: THREE.Scene) => {
101+
const exrLoader = new EXRLoader()
102+
103+
// exr from polyhaven.com
104+
exrLoader.load('round_platform_1k.exr', (exrTex) => {
105+
exrTex.mapping = THREE.EquirectangularReflectionMapping
106+
scene.environment = exrTex
107+
scene.background = exrTex
108+
109+
const groundProjection = new GroundedSkybox(exrTex, 5, 50)
110+
groundProjection.position.set(0, 5, 0)
111+
scene.add(groundProjection)
112+
})
113+
}
114+
115+
const addFisheyeGUI = (fisheye: Fisheye) => {
116+
const folder = gui.addFolder('Fisheye Settings')
117+
folder.add(params, 'fisheyeEnabled').name('enabled')
118+
folder
119+
.add(params, 'zoom', 0, 1, 0.01)
120+
.onChange((value: number) => {
121+
fisheye.zoom = value
122+
fisheye.onResize(window.innerWidth, window.innerHeight)
123+
})
124+
.name('zoom')
125+
folder
126+
.add(params, 'resolution', 256, 1024, 128)
127+
.onChange((value: number) => {
128+
fisheye.resolution = value
129+
fisheye.updateResolution(value)
130+
})
131+
.name('resolution')
132+
}
133+
134+
const setupRaycaster = (renderer: THREE.WebGLRenderer, camera: THREE.Camera, cylinderMeshes: THREE.Mesh[]) => {
135+
const raycaster = new THREE.Raycaster()
136+
const pointer = new THREE.Vector2()
137+
let pointerDownTime = 0
138+
139+
renderer.domElement.addEventListener('pointerdown', () => {
140+
pointerDownTime = performance.now()
141+
})
142+
143+
renderer.domElement.addEventListener('pointerup', (event) => {
144+
const pointerUpTime = performance.now()
145+
const timeDiff = pointerUpTime - pointerDownTime
146+
147+
// Only treat as click if time between down/up is short
148+
if (timeDiff < 200) {
149+
// Calculate pointer position in normalized device coordinates (-1 to +1)
150+
pointer.x = (event.offsetX / renderer.domElement.clientWidth) * 2 - 1
151+
pointer.y = -(event.offsetY / renderer.domElement.clientHeight) * 2 + 1
152+
153+
if (params.fisheyeEnabled) {
154+
fisheye.computeRaycastRayDirection(raycaster, pointer)
155+
} else {
156+
raycaster.setFromCamera(pointer, camera)
157+
}
158+
159+
const intersects = raycaster.intersectObjects(cylinderMeshes, true)
160+
if (intersects.length > 0) {
161+
// change color of first hit object
162+
const hit = intersects[0].object as THREE.Mesh
163+
hit.scale.y = THREE.MathUtils.randFloat(1, 5)
164+
const mat = hit.material as THREE.MeshStandardMaterial
165+
mat.color.setHSL(Math.random(), 1, 0.5)
166+
}
167+
}
168+
})
169+
}
170+
171+
FisheyeStory.storyName = 'Default'

‎README.md‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ import { pcss, ... } from '@pmndrs/vanilla'
7575
<li><a href="#portals">Portals</a></li>
7676
<ul>
7777
<li><a href="#meshportalmaterial">MeshPortalMaterial</a></li>
78+
<li><a href="#fisheye">Fisheye</a></li>
7879
</ul>
7980
</ul>
8081
</td>
@@ -1058,3 +1059,55 @@ portalMesh = new THREE.Mesh(portalGeometry, portalMaterial)
10581059
meshPortalMaterialApplySDF(portalMesh, 512, renderer) // 512 is SDF texture resolution
10591060
scene.add(portalMesh)
10601061
```
1062+
1063+
### Fisheye
1064+
1065+
[![storybook](https://img.shields.io/badge/-storybook-%23ff69b4)](https://pmndrs.github.io/drei-vanilla/?path=/story/portals-fisheye--fisheye-story)
1066+
1067+
[drei counterpart](https://drei.docs.pmnd.rs/portals/fisheye)
1068+
1069+
```ts
1070+
export type FisheyeProps = {
1071+
/** Zoom factor, 0..1, default:0 */
1072+
zoom?: number
1073+
/** Number of segments, default: 64 */
1074+
segments?: number
1075+
/** Cubemap resolution default: 896 */
1076+
resolution?: number
1077+
}
1078+
```
1079+
1080+
This class will take over rendering. It captures the scene into a cubemap from the camera's pov which is then projected onto a sphere. The sphere is then rendered to fill the screen. You can lower the resolution to increase performance. Six renders per frame are necessary to construct a full fisheye view, and since each face of the cubemap only takes a portion of the screen, full resolution is not necessary. You can also reduce the number of segments of the sphere (resulting in a more blocky sphere).
1081+
1082+
Usage:
1083+
1084+
```js
1085+
const fishEye = new Fisheye()
1086+
1087+
// on resize event
1088+
// make sure onResize is called once before first frame is rendered
1089+
onWindowResize(){
1090+
//...
1091+
fishEye.onResize(width, height)
1092+
}
1093+
1094+
// In animation loop, use fisheye.render() instead of renderer.render()
1095+
renderer.setAnimationLoop(() => {
1096+
//...
1097+
controls.update() // if using controls
1098+
1099+
// Use fisheye's rendering instead of normal rendering
1100+
fishEye.render(renderer, scene, camera) // replaces: renderer.render(scene, camera)
1101+
})
1102+
1103+
// To raycast through fisheye distortion,
1104+
// use computeRaycastRayDirection instead of raycaster.setFromCamera
1105+
const pointer = new THREE.Vector2()
1106+
raycast(){
1107+
//...
1108+
fishEye.computeRaycastRayDirection(raycaster, pointer)
1109+
const intersects = raycaster.intersectObjects(raycastMeshes)
1110+
//...
1111+
}
1112+
1113+
```

‎src/core/Fisheye.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import * as THREE from 'three'
2+
3+
export type FisheyeProps = {
4+
/** Zoom factor, 0..1, default:0 */
5+
zoom?: number
6+
/** Number of segments, default: 64 */
7+
segments?: number
8+
/** Cubemap resolution default: 896 */
9+
resolution?: number
10+
}
11+
12+
export class Fisheye {
13+
resolution: number
14+
segments: number
15+
zoom: number
16+
renderTarget: THREE.WebGLCubeRenderTarget
17+
sphereMesh: THREE.Mesh<THREE.SphereGeometry, THREE.MeshBasicMaterial>
18+
orthoCamera: THREE.OrthographicCamera
19+
cubeCamera: THREE.CubeCamera
20+
temp: {
21+
t: THREE.Vector3
22+
r: THREE.Quaternion
23+
s: THREE.Vector3
24+
e: THREE.Euler
25+
sphere: THREE.Sphere
26+
normal: THREE.Vector3
27+
normalMatrix: THREE.Matrix3
28+
}
29+
/**
30+
* Creates a new Fisheye camera renderer instance
31+
*/
32+
constructor({ resolution = 896, segments = 64, zoom = 0 }: FisheyeProps = {}) {
33+
this.resolution = resolution
34+
this.segments = segments
35+
this.zoom = zoom
36+
37+
this.renderTarget = this.createRenderTarget(resolution)
38+
const geometry = new THREE.SphereGeometry(1, segments, segments)
39+
const material = new THREE.MeshBasicMaterial({ envMap: this.renderTarget.texture })
40+
this.sphereMesh = new THREE.Mesh(geometry, material)
41+
42+
this.orthoCamera = new THREE.OrthographicCamera()
43+
this.cubeCamera = new THREE.CubeCamera(0.1, 1000, this.renderTarget)
44+
45+
this.temp = {
46+
t: new THREE.Vector3(),
47+
r: new THREE.Quaternion(),
48+
s: new THREE.Vector3(),
49+
e: new THREE.Euler(0, Math.PI, 0),
50+
sphere: new THREE.Sphere(),
51+
normal: new THREE.Vector3(),
52+
normalMatrix: new THREE.Matrix3(),
53+
}
54+
55+
this.onResize(100, 100)
56+
}
57+
58+
/**
59+
* @private
60+
* Creates a WebGL cube render target for capturing the 360° environment
61+
* @param resolution The resolution for each face of the cube texture
62+
* @returns A configured WebGLCubeRenderTarget with optimal settings for fisheye rendering
63+
*/
64+
createRenderTarget(resolution: number) {
65+
const rt = new THREE.WebGLCubeRenderTarget(resolution, {
66+
stencilBuffer: true,
67+
depthBuffer: true,
68+
generateMipmaps: true,
69+
flipY: true,
70+
type: THREE.HalfFloatType,
71+
})
72+
rt.texture.isRenderTargetTexture = false
73+
return rt
74+
}
75+
76+
/**
77+
* Updates the cube render target resolution and refreshes associated materials
78+
* Disposes of the old render target to prevent memory leaks
79+
* @param resolution New resolution for the cube texture faces
80+
*/
81+
updateResolution(resolution: number) {
82+
if (resolution === this.renderTarget.width) return
83+
this.renderTarget.dispose()
84+
this.renderTarget = this.createRenderTarget(resolution)
85+
this.cubeCamera.renderTarget = this.renderTarget
86+
this.sphereMesh.material.envMap = this.renderTarget.texture
87+
this.sphereMesh.material.needsUpdate = true
88+
}
89+
90+
/**
91+
* Handles viewport resize by updating camera projection and fisheye sphere scaling
92+
* Recalculates the orthographic camera bounds and sphere radius based on new dimensions
93+
* @param width New viewport width in pixels
94+
* @param height New viewport height in pixels
95+
*/
96+
onResize(width: number, height: number) {
97+
const w = width,
98+
h = height
99+
this.orthoCamera.position.set(0, 0, 100)
100+
this.orthoCamera.zoom = 100
101+
this.orthoCamera.left = w / -2
102+
this.orthoCamera.right = w / 2
103+
this.orthoCamera.top = h / 2
104+
this.orthoCamera.bottom = h / -2
105+
106+
this.orthoCamera.updateProjectionMatrix()
107+
108+
const radius = (Math.sqrt(w * w + h * h) / 100) * (0.5 + this.zoom / 2)
109+
this.sphereMesh.scale.setScalar(radius)
110+
this.temp.sphere.radius = radius
111+
}
112+
113+
/**
114+
* Computes the correct ray direction for raycasting through the fisheye projection
115+
* Transforms 2D screen coordinates into 3D world space ray for accurate object picking
116+
* @param raycaster The Three.js raycaster to modify with the computed ray
117+
* @param pointer Normalized pointer coordinates (-1 to 1 range)
118+
* @returns void - modifies the raycaster's ray direction in place
119+
*/
120+
computeRaycastRayDirection(raycaster: THREE.Raycaster, pointer: THREE.Vector2) {
121+
/**
122+
* Event compute by Garrett Johnson https://twitter.com/garrettkjohnson
123+
* https://discourse.threejs.org/t/how-to-use-three-raycaster-with-a-sphere-projected-envmap/56803/10
124+
*/
125+
126+
// Raycast from the render camera to the sphere and get the surface normal
127+
// of the point hit in world space of the sphere scene
128+
// We have to set the raycaster using the ortho cam and pointer
129+
// to perform sphere intersections.
130+
131+
const { orthoCamera: orthoC, temp, cubeCamera: cubeCamera } = this
132+
const { normal, normalMatrix, sphere: sph } = temp
133+
raycaster.setFromCamera(pointer, orthoC)
134+
if (!raycaster.ray.intersectSphere(sph, normal)) return
135+
else normal.normalize()
136+
// Get the matrix for transforming normals into world space
137+
normalMatrix.getNormalMatrix(cubeCamera.matrixWorld)
138+
// Get the ray
139+
cubeCamera.getWorldPosition(raycaster.ray.origin)
140+
raycaster.ray.direction.set(0, 0, 1).reflect(normal)
141+
raycaster.ray.direction.x *= -1 // flip across X to accommodate the "flip" of the env map
142+
raycaster.ray.direction.applyNormalMatrix(normalMatrix).multiplyScalar(-1)
143+
}
144+
145+
/**
146+
* Renders the fisheye effect by capturing a 360° cubemap and projecting it onto a sphere
147+
* Two-pass rendering: first captures the scene to cubemap, then renders the fisheye sphere
148+
* @param renderer The WebGL renderer to use for rendering
149+
* @param scene The 3D scene to capture in the fisheye view
150+
* @param camera The source camera whose position and orientation to use for the cubemap capture
151+
*/
152+
render(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera) {
153+
// copy original camera coords to cube camera
154+
camera.matrixWorld.decompose(this.temp.t, this.temp.r, this.temp.s)
155+
this.cubeCamera.position.copy(this.temp.t)
156+
this.cubeCamera.quaternion.setFromEuler(this.temp.e).premultiply(this.temp.r)
157+
158+
this.cubeCamera.update(renderer, scene) // render the cubemap
159+
renderer.render(this.sphereMesh, this.orthoCamera) // render the fisheye
160+
}
161+
}

0 commit comments

Comments
 (0)