I can't reproduce your problem. The "flatness" variable works exactly the way I expected it to. Here's the code I used:
load image "stone wall.png", 1
flatness = 172
makeNormalMap(1, 2, flatness) ` use source image 1 to create new image 2 with flatness in range 129-255
paste image 2, 0, 0
set cursor 320, 20
print image width(2), " ", image height(2)
wait key
save image "test nMap "+str$(flatness)+".png", 2
end
function makeNormalMap(sourceImageID, normalMapID, flatness)
` assumes a seamless source so this code wraps around the edges
c as dword
r as byte
g as byte
b as byte
make memblock from image 1, sourceImageID
make memblock from image 2, sourceImageID ` this will be overwritten with the new image
height = image height(sourceImageID) : height1 = height - 1
width = image width(sourceImageID) : width1 = width - 1
for h = 0 to height1
for w = 0 to width1
if w > 0
c = memblock dword(1,(h*width+w-1)*4+12)
else
c = memblock dword(1,(h*width+width1)*4+12)
endif
r = RGBB(c)
if w < width1
c = memblock dword(1,(h*width+w+1)*4+12)
else
c = memblock dword(1,h*width*4+12)
endif
r = (r-RGBB(c)+255)/2
if (h > 0)
c = memblock dword(1,((h-1)*width+w)*4+12)
else
c = memblock dword(1,(height1*width+w)*4+12)
endif
g = RGBB(c)
if h < height1
c = memblock dword(1,((h+1)*width+w)*4+12)
else
c = memblock dword(1,w*4+12)
endif
g = (g-RGBB(c)+255)/2;
write memblock dword 2,(h*width+w)*4+12, RGB(r,g,flatness)
next w
next h
make image from memblock normalMapID, 2
delete memblock 1
delete memblock 2
endfunction
Here are some typical screenshots using one of my shaders in Dark Shader:
Edit Forgot to mention that the shader normalizes the input value from the normal map as I discussed in an earlier post.