I'm trying to implement a 2D outline shader in OpenGL ES2.0 for iOS. It is insanely slow. As in 5fps slow. I've tracked it down to the texture2D() calls. However, without those any convolution shader is undoable. I've tried using lowp instead of mediump, but with that everything is just black, although it does give another 5fps, but it's still unusable.
Here is my fragment shader.
varying mediump vec4 colorVarying;
varying mediump vec2 texCoord;
uniform bool enableTexture;
uniform sampler2D texture;
uniform mediump float k;
void main() {
const mediump float step_w = 3.0/128.0;
const mediump float step_h = 3.0/128.0;
const mediump vec4 b = vec4(0.0, 0.0, 0.0, 1.0);
const mediump vec4 one = vec4(1.0, 1.0, 1.0, 1.0);
mediump vec2 offset[9];
mediump float kernel[9];
offset[0] = vec2(-step_w, step_h);
offset[1] = vec2(-step_w, 0.0);
offset[2] = vec2(-step_w, -step_h);
offset[3] = vec2(0.0, step_h);
offset[4] = vec2(0.0, 0.0);
offset[5] = vec2(0.0, -step_h);
offset[6] = vec2(step_w, step_h);
offset[7] = vec2(step_w, 0.0);
offset[8] = vec2(step_w, -step_h);
kernel[0] = kernel[2] = kernel[6] = kernel[8] = 1.0/k;
kernel[1] = kernel[3] = kernel[5] = kernel[7] = 2.0/k;
kernel[4] = -16.0/k;
if (enableTexture) {
mediump vec4 sum = vec4(0.0);
for (int i=0;i<9;i++) {
mediump vec4 tmp = texture2D(texture, texCoord + offset[i]);
sum += tmp * kernel[i];
}
gl_FragColor = (sum * b) + ((one-sum) * texture2D(texture, texCoord));
} else {
gl_FragColor = colorVarying;
}
}
This is unoptimized, and not finalized, but I need to bring up performance before continuing on. I've tried replacing the texture2D() call in the loop with just a solid vec4 and it runs no problem, despite everything else going on.
How can I optimize this? I know it's possible because I've seen way more involved effects in 3D running no problem. I can't see why this is causing any trouble at all.
I've done this exact thing myself, and I see several things that could be optimized here.
First off, I'd remove the enableTexture conditional and instead split your shader into two programs, one for the true state of this and one for false. Conditionals are very expensive in iOS fragment shaders, particularly ones that have texture reads within them.
Second, you have nine dependent texture reads here. These are texture reads where the texture coordinates are calculated within the fragment shader. Dependent texture reads are very expensive on the PowerVR GPUs within iOS devices, because they prevent that hardware from optimizing texture reads using caching, etc. Because you are sampling from a fixed offset for the 8 surrounding pixels and one central one, these calculations should be moved up into the vertex shader. This also means that these calculations won't have to be performed for each pixel, just once for each vertex and then hardware interpolation will handle the rest.
Third, for() loops haven't been handled all that well by the iOS shader compiler to date, so I tend to avoid those where I can.
As I mentioned, I've done convolution shaders like this in my open source iOS GPUImage framework. For a generic convolution filter, I use the following vertex shader:
attribute vec4 position;
attribute vec4 inputTextureCoordinate;
uniform highp float texelWidth;
uniform highp float texelHeight;
varying vec2 textureCoordinate;
varying vec2 leftTextureCoordinate;
varying vec2 rightTextureCoordinate;
varying vec2 topTextureCoordinate;
varying vec2 topLeftTextureCoordinate;
varying vec2 topRightTextureCoordinate;
varying vec2 bottomTextureCoordinate;
varying vec2 bottomLeftTextureCoordinate;
varying vec2 bottomRightTextureCoordinate;
void main()
{
gl_Position = position;
vec2 widthStep = vec2(texelWidth, 0.0);
vec2 heightStep = vec2(0.0, texelHeight);
vec2 widthHeightStep = vec2(texelWidth, texelHeight);
vec2 widthNegativeHeightStep = vec2(texelWidth, -texelHeight);
textureCoordinate = inputTextureCoordinate.xy;
leftTextureCoordinate = inputTextureCoordinate.xy - widthStep;
rightTextureCoordinate = inputTextureCoordinate.xy + widthStep;
topTextureCoordinate = inputTextureCoordinate.xy - heightStep;
topLeftTextureCoordinate = inputTextureCoordinate.xy - widthHeightStep;
topRightTextureCoordinate = inputTextureCoordinate.xy + widthNegativeHeightStep;
bottomTextureCoordinate = inputTextureCoordinate.xy + heightStep;
bottomLeftTextureCoordinate = inputTextureCoordinate.xy - widthNegativeHeightStep;
bottomRightTextureCoordinate = inputTextureCoordinate.xy + widthHeightStep;
}
and the following fragment shader:
precision highp float;
uniform sampler2D inputImageTexture;
uniform mediump mat3 convolutionMatrix;
varying vec2 textureCoordinate;
varying vec2 leftTextureCoordinate;
varying vec2 rightTextureCoordinate;
varying vec2 topTextureCoordinate;
varying vec2 topLeftTextureCoordinate;
varying vec2 topRightTextureCoordinate;
varying vec2 bottomTextureCoordinate;
varying vec2 bottomLeftTextureCoordinate;
varying vec2 bottomRightTextureCoordinate;
void main()
{
mediump vec4 bottomColor = texture2D(inputImageTexture, bottomTextureCoordinate);
mediump vec4 bottomLeftColor = texture2D(inputImageTexture, bottomLeftTextureCoordinate);
mediump vec4 bottomRightColor = texture2D(inputImageTexture, bottomRightTextureCoordinate);
mediump vec4 centerColor = texture2D(inputImageTexture, textureCoordinate);
mediump vec4 leftColor = texture2D(inputImageTexture, leftTextureCoordinate);
mediump vec4 rightColor = texture2D(inputImageTexture, rightTextureCoordinate);
mediump vec4 topColor = texture2D(inputImageTexture, topTextureCoordinate);
mediump vec4 topRightColor = texture2D(inputImageTexture, topRightTextureCoordinate);
mediump vec4 topLeftColor = texture2D(inputImageTexture, topLeftTextureCoordinate);
mediump vec4 resultColor = topLeftColor * convolutionMatrix[0][0] + topColor * convolutionMatrix[0][1] + topRightColor * convolutionMatrix[0][2];
resultColor += leftColor * convolutionMatrix[1][0] + centerColor * convolutionMatrix[1][1] + rightColor * convolutionMatrix[1][2];
resultColor += bottomLeftColor * convolutionMatrix[2][0] + bottomColor * convolutionMatrix[2][1] + bottomRightColor * convolutionMatrix[2][2];
gl_FragColor = resultColor;
}
The texelWidth and texelHeight uniforms are the inverse of the width and height of the input image, and the convolutionMatrix uniform specifies the weights for the various samples in your convolution.
On an iPhone 4, this runs in 4-8 ms for a 640x480 frame of camera video, which is good enough for 60 FPS rendering at that image size. If you just need to do something like edge detection, you can simplify the above, convert the image to luminance in a pre-pass, then only sample from one color channel. That's even faster, at about 2 ms per frame on the same device.
The only way I know of reducing time taken in this shader is by reducing the number of texture fetches. Since your shader samples textures from equally spaced points about the center pixels and linearly combines them, you could reduce the number of fetches by making use of the GL_LINEAR mode availbale for texture sampling.
Basically instead of sampling at every texel, sample in between a pair of texels to directly get a linearly weighted sum.
Let us call the sampling at offset (-stepw,-steph) and (-stepw,0) as x0 and x1 respectively. Then your sum is
sum = x0*k0 + x1*k1
Now instead if you sample in between these two texels, at a distance of
k0/(k0+k1) from x0 and therefore k1/(k0+k1) from x1, then the GPU will perform the linear weighting during the fetch and give you,
y = x1*k1/(k0+k1) + x0*k0/(k1+k0)
Thus sum can be calculated as
sum = y*(k0 + k1) from just one fetch!
If you repeat this for the other adjacent pixels, you will end up doing 4 texture fetches for each of the adjacent offsets, and one extra texture fetch for the center pixel.
The link explains this much better
Related
In my code, I'm mixing two textures. I want to position a texture at any place on the plane but when I add an offset to the texture UV XY coordinate the image just gets stretched.
offsetText1 = vec2(0.1,0.1);
vec4 displacement = texture2D(utexture1,vUv+offsetText1);
How do I move the texture to any position without stretching it?
VERTEX SHADER:
varying vec2 vUv;
uniform sampler2D utexture1;
uniform sampler2D utexture2;
varying vec2 offsetText1;
void main() {
offsetText1 = vec2(0.1,0.1);
vUv = uv;
vec4 modelPosition = modelMatrix * vec4(position, 1.0);
vec4 displacement = texture2D(utexture1,vUv+offsetText1);
vec4 displacement2 = texture2D(utexture2,vUv);
modelPosition.z += displacement.r*1.0;
modelPosition.z += displacement2.r*40.0;
gl_Position = projectionMatrix * viewMatrix * modelPosition;
}
FRAGMENT SHADER:
#ifdef GL_ES
precision highp float;
#endif
uniform sampler2D utexture1;
uniform sampler2D utexture2;
varying vec2 vUv;
varying vec2 offsetText1;
void main() {
vec3 c;
vec4 Ca = texture2D(utexture1,vUv+offsetText1 );
vec4 Cb = texture2D(utexture2,vUv);
c = Ca.rgb * Ca.a + Cb.rgb * Cb.a * (2.0 - Ca.a);
gl_FragColor = vec4(c, 1.0);
}
image with offsetText1 = vec2(0.0,0.0);
no stretching
image with offsetText1 = vec2(0.1,0.1); image is being stretched from the top right corner.
stretching
That's the behavior of textures. They extend in the range from [0, 1], so when you go beyond 1 or below 0, they'll "wrap". You need to tell it what to do when wrapping. Do you want it to repeat, stretch, or mirror?
You could establish this with the texture.wrapS and .wrapT properties, which accepts one of 3 values:
THREE.RepeatWrapping
THREE.ClampToEdgeWrapping
THREE.MirroredRepeatWrapping
If you want to just show white where the texture extends out of bounds, then you'd have to do that programmatically in your shader code. Here's some pseudocode:
if (uv < 0 || > 1)
color = white
I'm implementing the xToon shader(pdf) in glsl to use as a shader with Three.js.
I'm getting some rendering artifacts, and I think the problem is due to webgl strangeness that I am not knowledgable about, perhaps relating to a Nan or Inf or something... I'm pulling my hair out.
I'll include the complete fragment and vertex shaders below, but I think this is the offending code located in the fragment shader:
....
vec3 n = normalize(vNormal);
vec3 l = normalize(lightDir);
float d = dot(n, l) * 0.5 + 0.5;
//vec2 texLookUp = vec2(d, loa);
vec2 texLookUp = vec2(d, 0.055);
vec4 dColor = texture2D(texture, texLookUp);
gl_FragColor = dColor;
....
To the best of my debugging efforts there seems to be some problem with using the value d as a component of the texture look up vector. This code produces these strange artifacts:
There shouldn't be those yellow "lines" on those contours...
As you may have noted, I'm not actually using the "loa" value in this code. For a while I thought that this problem was in the way I was calculating loa, but it seems that this bug is independent of loa.
Any help would be much appreciated!
The fragment shader:
uniform vec3 lightDir;
uniform sampler2D texture;
varying vec3 vNormal;
varying vec3 vPosition;
varying vec2 vUv;
// loa calculation for texture lookup
varying highp float loa;
void main() {
vec3 n = normalize(vNormal);
vec3 l = normalize(lightDir);
float d = dot(n, l) * 0.5 + 0.5;
//vec2 texLookUp = vec2(d, loa);
vec2 texLookUp = vec2(d, 0.055);
vec4 dColor = texture2D(texture, texLookUp);
gl_FragColor = dColor;
}
And the vertex shader:
uniform vec3 cameraPos;
uniform vec3 lightDir;
uniform vec3 focalPos;
uniform float inflate;
uniform float zmin;
uniform float r;
varying vec3 vNormal;
varying vec2 vUv;
varying float loa;
void main() {
vec3 n = normalize(normal);
// euclidiean distance to point from camera pos
float depth = length(cameraPos - position);
// 1. detail mapping correcting for perspective projection
float z = depth / zmin;
loa = 1.0 - (log2(z)/log2(r));
loa = clamp(loa, 0.055, 0.9);
vNormal = n;
vUv = uv;
gl_Position = projectionMatrix * modelViewMatrix * vec4(normal * inflate + position, 1.0 );
}
I solved the problem by setting the texture to ClampToEdgeWrapping instead of RepeatWrapping. I was led to this answer by this stack overflow question:
Using floor() function in GLSL when sampling a texture leaves glitch
The solution is explained very well in this blog post:
http://webglfundamentals.org/webgl/lessons/webgl-3d-textures.html
And the functions to deal with this in THREEjs are members of the Texture and are explained in the THREEjs docs here.
Also I needed to set the min filter to Nearest to fully get rid of the artifacts.
My phong fragment shader is not shading anything, just making everything black.
This is my fragment shader
precision mediump float;
varying vec3 vposition;
varying vec3 vnormal;
varying vec4 vcolor;
varying vec3 veyePos;
void main() {
vec3 lightPos = vec3(0,0,0);
vec4 s = normalize(vec4(lightPos,1) - vec4(veyePos,1));
vec4 r = reflect(-s,vec4(vnormal, 1));
vec4 v = normalize(-vec4(veyePos, 1));
float spec = max( dot(v,r),0.0 );
float diff = max(dot(vec4(vnormal,1),s),0.0);
vec3 diffColor = diff * vec3(1,0,0);
vec3 specColor = pow(spec,3.0) * vec3(1,1,1);
vec3 ambientColor = vec3(0.1,0.1,0.1);
gl_FragColor = vec4(diffColor + 0.5 * specColor + ambientColor, 1);
}
This is my vertex shader
uniform mat4 uMVPMatrix;
uniform mat4 uMVMatrix;
uniform vec3 eyePos;
attribute vec4 aPosition;
attribute vec4 aColor;
attribute vec4 aNormal;
varying vec4 vcolor;
varying vec3 vposition;
varying vec3 vnormal;
varying vec3 veyePos;
void main() {
mat4 normalMat = transpose(inverse(uMVMatrix));
vec4 vertPos4 = uMVMatrix * vec4(vec3(aPosition), 1.0);
vposition = vec3(vertPos4) / vertPos4.w;
vcolor = aColor;
veyePos = eyePos;
vnormal = vec3(uMVMatrix * vec4(vec3(aNormal),0.0));
gl_Position = uMVPMatrix * aPosition;
}
MVMatrix is model-view matrix
MVPMatrix is model-view-projection matrix
first of all your lighting equations are incorrect:
vector s that you use to calculate the diffuse color should be a unit vector that originates at your vertex (vposition) towards your light. so it would be
s = normalize(lightPos - vposition)
also lightPos should be given in camera space and not in world space (so you should multiply it by your MV matrix)
vector r is the reflection of s around the normal so i dont understand why you take -s also the normal there should be in non-homegenous coordinates so it would be:
r = reflect(s,vnormal)
and finally v is the viewing ray (multiplied by -1) so it should be the vector that originates at vposition and goes towards eyepos.
v = normalize(veyepos - vposition)
also in your vertex shader veyepos (assuming it is the position of your camera) should not be varying (should be flat variable) because you dont want to interpolate it.
in your vertex shader you calculate normalMat but you forgot to use it when calculating your normals in camera space.
also normalMat should be mat3 because it is the inverse transpose of the
3by3 block of your MV matrix originating at (0,0)
** in order to be efficient you should calculate your normalMat on the cpu and pass it as a uniform to your vertex shader
hope this helps
I've been trying to implement a basic fog function, and some resources I've found have led me to try a linear fog function that my understanding has led me to believe is based on the original fixed function fog equation (though I could be wrong on that). However, when I try to implement it, all my surfaces become black. I'm pretty sure my problem lies in how I (attempt to) compute the eye distance, as the rest of the equation is pretty straightforward.
My vertex shader:
precision mediump float;
attribute vec4 position;
attribute vec3 normal;
attribute vec2 texCoordIn;
varying vec3 eyeNormal;
varying vec4 eyePos;
varying vec2 texCoordOut;
varying float eyeDistance;
uniform mat4 modelViewProjectionMatrix;
uniform mat4 modelViewMatrix;
uniform mat3 normalMatrix;
uniform vec3 cameraPosition;
void main()
{
eyeNormal = (normalMatrix * normal);
eyePos = modelViewMatrix * position;
eyeDistance = sqrt((eyePos.x - cameraPosition.x) * (eyePos.x - cameraPosition.x) + (eyePos.y - cameraPosition.y) * (eyePos.y - cameraPosition.y) + (eyePos.z - cameraPosition.z) * (eyePos.z - cameraPosition.z));
texCoordOut = texCoordIn;
gl_Position = modelViewProjectionMatrix * position;
}
My fragment shader:
precision mediump float;
varying vec3 eyeNormal;
varying vec4 eyePos;
varying vec2 texCoordOut;
varying float eyeDistance;
uniform sampler2D texture;
uniform vec3 flashlightPosition;
uniform vec3 diffuseLightPosition;
uniform vec4 diffuseComponent;
uniform float shininess;
uniform vec4 specularComponent;
uniform vec4 ambientComponent;
uniform float maxFogDistance;
uniform float minFogDistance;
uniform vec4 fogColor;
void main()
{
vec4 ambient = ambientComponent;
vec3 N = normalize(eyeNormal);
float nDotVP = max(0.0, dot(N, normalize(diffuseLightPosition)));
vec4 diffuse = diffuseComponent * nDotVP;
vec3 E = normalize(-eyePos.xyz);
vec3 L = normalize(flashlightPosition - eyePos.xyz);
vec3 H = normalize(L+E);
float Ks = pow(max(dot(N, H), 0.0), shininess);
vec4 specular = Ks*specularComponent;
if( dot(L, N) < 0.0 ) {
specular = vec4(0.0, 0.0, 0.0, 1.0);
}
float factor = (maxFogDistance - eyeDistance) / (maxFogDistance - minFogDistance);
factor = clamp(factor, 0.0, 1.0);
vec4 finalFogColor = factor * fogColor;
vec4 finalColor = (ambient + specular) * texture2D(texture, texCoordOut);
gl_FragColor = finalColor * factor + fogColor * (1.0 - factor);
gl_FragColor.a = 1.0;
}
Additionally, the minimum fog distance has been set to 1f and the maximum to 10f (my movement space is quite small, currently 5x5, so this should be appropriate) and the fog color is set to white.
Any help anybody could provide would be much appreciated. I can post additional code if the problem isn't in the shaders (though, I can't figure anywhere else it would be, the other aspects of the shaders worked fine before this).
I have a 3x3 homography matrix that works correctly with OpenCV's warpPerspective, but I need to do the warping on GPU for performance reasons. What is the best approach? I tried multiplying in the vertex shader to get the texture coordinates and then render a quad, but I get strange distortions. I'm not sure if it's the interpolation not working as I expect. Attaching output for comparison (it involves two different, but close enough shots).
Absolute difference of warp and other image from GPU:
Composite of warp and other image in OpenCV:
EDIT:
Following are my shaders: the task is image rectification (making epilines become scanlines) + absolute difference.
// Vertex Shader
static const char* warpVS = STRINGIFY
(
uniform highp mat3 homography1;
uniform highp mat3 homography2;
uniform highp int width;
uniform highp int height;
attribute highp vec2 position;
varying highp vec2 refTexCoords;
varying highp vec2 curTexCoords;
highp vec2 convertToTexture(highp vec3 pixelCoords) {
pixelCoords /= pixelCoords.z; // need to project
pixelCoords /= vec3(float(width), float(height), 1.0);
pixelCoords.y = 1.0 - pixelCoords.y; // origin is in bottom left corner for textures
return pixelCoords.xy;
}
void main(void)
{
gl_Position = vec4(position / vec2(float(width) / 2.0, float(height) / 2.0) - vec2(1.0), 0.0, 1.0);
gl_Position.y = -gl_Position.y;
highp vec3 initialCoords = vec3(position, 1.0);
refTexCoords = convertToTexture(homography1 * initialCoords);
curTexCoords = convertToTexture(homography2 * initialCoords);
}
);
// Fragment Shader
static const char* warpFS = STRINGIFY
(
varying highp vec2 refTexCoords;
varying highp vec2 curTexCoords;
uniform mediump sampler2D refTex;
uniform mediump sampler2D curTex;
uniform mediump sampler2D maskTex;
void main(void)
{
if (texture2D(maskTex, refTexCoords).r == 0.0) {
discard;
}
if (any(bvec4(curTexCoords[0] < 0.0, curTexCoords[1] < 0.0, curTexCoords[0] > 1.0, curTexCoords[1] > 1.0))) {
discard;
}
mediump vec4 referenceColor = texture2D(refTex, refTexCoords);
mediump vec4 currentColor = texture2D(curTex, curTexCoords);
gl_FragColor = vec4(abs(referenceColor.r - currentColor.r), 1.0, 0.0, 1.0);
}
);
I think you just need to do the projection per pixel. Make refTexCoords and curTexCoords at least vec3, then do the /z in the pixel shader before texture lookup. Even better use the textureProj GLSL instruction.
You want to do everything that is linear in the vertex shader, but things like projection need to be done in the fragment shader per pixel.
This link might help with some background: http://www.reedbeta.com/blog/2012/05/26/quadrilateral-interpolation-part-1/