Vertex from animated mesh

How I can get vertex from animated mesh?

TargetControl.prototype.getVertexPos = function(entity, vertexNum) {
    var vertexes = [];
    entity.model.meshInstances[0].mesh.getPositions(vertexes);
    var normals = [];
    entity.model.meshInstances[0].mesh.getNormals(normals);
    var index = vertexNum * 3;
    var vertex = new pc.Vec3(vertexes[index], vertexes[index+1], vertexes[index+2]).add(entity.getLocalPosition());
    var normal = new pc.Vec3(normals[index], normals[index+1], normals[index+2]);
    return { vertex: vertex, normal: normal };
};

TargetControl.prototype.update = function(dt) {
    var v = this.getVertexPos(this.controlEntity, this.vertexNum);
    this.entity.setLocalPosition(v.vertex)
};

returns only static position

Calling @mvaligursky, he may be able to help on this.

@mvaligursky can You help?

@ray is putting some simpler version of code snippets together - he’s done skinning similar to what you need for the cloth demo few weeks ago.

How can help cloth simulation for getting vertex from nonprocedural animated mesh?

cloth simulation uses skinned mesh as an input

This function should return a skinned vertex and normal:

TargetControl.prototype.getSkinnedVertexPos = function(entity, vertexNum) {
    var vertexes = [];
    entity.model.meshInstances[0].mesh.getPositions(vertexes);
    var normals = [];
    entity.model.meshInstances[0].mesh.getNormals(normals);
    var boneMatrices = entity.model.meshInstances[0].skinInstance.matrices;  
    var boneWeights = [];
    entity.model.meshInstances[0].mesh.getVertexStream(pc.SEMANTIC_BLENDWEIGHT, boneWeights);
    var boneIndices = [];
    entity.model.meshInstances[0].mesh.getVertexStream(pc. SEMANTIC_BLENDINDICES, boneIndices);
    var index = vertexNum * 3;
    var boneWeight = new pc.Vec4(boneWeights[index], boneWeights[index+1], boneWeights[index+2], boneWeights[index+3]);
    var boneIndex = new pc.Vec4(boneIndices[index], boneIndices[index+1], boneIndices[index+2], boneIndices[index+3]);
    var unSkinnedVert = new pc.Vec4(vertexes[index], vertexes[index+1], vertexes[index+2], 1.0);
    var unSkinnedNorm = new pc.Vec4(normals[index], normals[index+1], normals[index+2], 0.0);
    var skinMat = new pc.Mat4();
    for (var m = 0; m < 16; m++) {
        skinMat.data[m] = boneMatrices[boneIndex.x].data[m]*boneWeight.x;
        skinMat.data[m] += boneMatrices[boneIndex.y].data[m]*boneWeight.y;
        skinMat.data[m] += boneMatrices[boneIndex.z].data[m]*boneWeight.z;
        skinMat.data[m] += boneMatrices[boneIndex.w].data[m]*boneWeight.w;
    }
    var skinnedVert = new pc.Vec4();
    var skinnedNorm = new pc.Vec4();
    skinnedVert = skinMat.transformVec4(unSkinnedVert);
    skinnedNorm = skinMat.transformVec4(unSkinnedNorm);
    var vertex = new pc.Vec3(skinnedVert.x, skinnedVert.y, skinnedVert.z).add(entity.getLocalPosition());
    var normal = new pc.Vec3(skinnedNorm.x, skinnedNorm.y, skinnedNorm.z);
    return { vertex: vertex, normal: normal };
};

NB As can be seen it is quite expensive to apply skinning to vertices and normals on in javascript on CPU - it is done much more efficiently on GPU in shaders - and so it should avoided if an alternative exist e.g hitboxes(https://en.wiktionary.org/wiki/hitbox)

1 Like

if you need to call this every frame, ideally you call these one time and keep the results:
getPositions
getNormals
getVertexStream(pc.SEMANTIC_BLENDWEIGHT
getVertexStream(pc. SEMANTIC_BLENDINDICES

and call the rest each frame, that is not too expensive for just few vertices.

2 Likes
TargetControl.prototype.getSkinnedVertexPos = function(entity, vertexNum) {
    if (!this.vertexes) { 
        this.vertexes = [];
        entity.model.meshInstances[0].mesh.getPositions(this.vertexes);
    }
    if (!this.normals) {
        this.normals = [];
        entity.model.meshInstances[0].mesh.getNormals(this.normals);
    }
    if(!this.boneWeights) {
        this.boneWeights = [];
        entity.model.meshInstances[0].mesh.getVertexStream(pc.SEMANTIC_BLENDWEIGHT, this.boneWeights);
    }
    if (!this.boneIndices) {
        this.boneIndices = [];
        entity.model.meshInstances[0].mesh.getVertexStream(pc. SEMANTIC_BLENDINDICES, this.boneIndices);
    }
    var boneMatrices = entity.model.meshInstances[0].skinInstance.matrices;
    
    var index = vertexNum * 3;
    var boneWeight = new pc.Vec4(this.boneWeights[index], this.boneWeights[index+1], this.boneWeights[index+2], this.boneWeights[index+3]);
    var boneIndex  = new pc.Vec4(this.boneIndices[index], this.boneIndices[index+1], this.boneIndices[index+2], this.boneIndices[index+3]);
    var unSkinnedVert = new pc.Vec4(this.vertexes[index], this.vertexes[index+1], this.vertexes[index+2], 1.0);
    var unSkinnedNorm = new pc.Vec4(this.normals[index],  this.normals[index+1],  this.normals[index+2], 0.0);
    var skinMat = new pc.Mat4();
    for (var m = 0; m < 16; m++) {
        skinMat.data[m] = boneMatrices[boneIndex.x].data[m]*boneWeight.x;
        skinMat.data[m] += boneMatrices[boneIndex.y].data[m]*boneWeight.y;
        skinMat.data[m] += boneMatrices[boneIndex.z].data[m]*boneWeight.z;
        skinMat.data[m] += boneMatrices[boneIndex.w].data[m]*boneWeight.w;
    }
    var skinnedVert = new pc.Vec4();
    var skinnedNorm = new pc.Vec4();
    skinnedVert = skinMat.transformVec4(unSkinnedVert);
    skinnedNorm = skinMat.transformVec4(unSkinnedNorm);
    var vertex = new pc.Vec3(skinnedVert.x, skinnedVert.y, skinnedVert.z);
    var normal = new pc.Vec3(skinnedNorm.x, skinnedNorm.y, skinnedNorm.z);
    return { vertex: vertex, normal: normal };
};
1 Like

Result is world or local position & normal?

It should be the same space as the un-skinned version of the function - which looks like object space with a local translational offset?


Incorrect ((
with 0.1 scale of result vector.
Red line to show target object. It must be on water mesh.

I’m pretty sure this is not correct. Arrays have 3 elements for position and normal, but 4 elements for bone weights and bone indices.

Corrected this. Not working. This is “unofficial API” or undocumented?

TargetControl.prototype.getSkinnedVertexPos = function(entity, vertexNum) {
    var boneMatrices = entity.model.meshInstances[0].skinInstance.matrices;
    var index3 = vertexNum * 3;
    var index4 = vertexNum * 4;
    var boneWeight = new pc.Vec4(this.boneWeights[index4], this.boneWeights[index4+1], this.boneWeights[index4+2], this.boneWeights[index4+3]);
    var boneIndex  = new pc.Vec4(this.boneIndices[index4], this.boneIndices[index4+1], this.boneIndices[index4+2], this.boneIndices[index4+3]);
    var unSkinnedVert = new pc.Vec4(this.vertexes[index3], this.vertexes[index3+1], this.vertexes[index3+2], 1.0);
    var unSkinnedNorm = new pc.Vec4(this.normals[index3],  this.normals[index3+1],  this.normals[index3+2], 0.0);
    var skinMat = new pc.Mat4();
    for (var m = 0; m < 16; m++) {
        skinMat.data[m]  = boneMatrices[boneIndex.x].data[m] * boneWeight.x;
        skinMat.data[m] += boneMatrices[boneIndex.y].data[m] * boneWeight.y;
        skinMat.data[m] += boneMatrices[boneIndex.z].data[m] * boneWeight.z;
        skinMat.data[m] += boneMatrices[boneIndex.w].data[m] * boneWeight.w;
    }
    var skinnedVert = new pc.Vec4();
    var skinnedNorm = new pc.Vec4();
    skinnedVert = skinMat.transformVec4(unSkinnedVert);
    skinnedNorm = skinMat.transformVec4(unSkinnedNorm);
    var vertex = new pc.Vec3(skinnedVert.x, skinnedVert.y, skinnedVert.z);
    var normal = new pc.Vec3(skinnedNorm.x, skinnedNorm.y, skinnedNorm.z);
    return { vertex: vertex, normal: normal };
};

Why boneIndex & boneWeight is Vec4? This must be int & float. I can’t understand ((

webgl1 device do not support integer types as vertex formats, and so floats is used everywhere.

It’s all official and documented. mesh.getVertexStream returns you array of data stored in mesh. But each mesh can store different data. Even position can be Vec3 or Vec2 or Vec4 or a single float (even though Vec3 is used for all 3d meshes in reality). It’s flexible.

Ideally you’d read elements in VertexFormat which is stored in VertexBuffer to understand the format of mesh data. But if you use typical 3d mesh imported into playcanvas, you can read out bone weights and bone vertices as array of floats with 4 floats per vertex.

1 Like

also, instead of transformVec4, the code should use faster transformPoint and transformVector (not that it would make the result different). That would give you Vec3 result directly and save some multiplies inside.

@ray - any idea why it still doesn’t work?
@KpoKec - when you say it doesn’t work … can you see what part? Is it position or normal or both? Do you transform results from local to world space if needed? Debug lines would need world space positions.

Screenshot above. Using only position, normal not implemented now.
I decided to abandon this option and make an animation. I just need animate position/rotation of some object over animated water mesh (include collider position animation).

They can be considered as individual numbers but when skinning is done in shaders on GPU they are often accessed as vec4s mainly for performance / convenience - as @mvaligursky said, feel free to refactor the code to make more sense to you - its more intended as a guide for how to do CPU skinning and it may not work without some debugging.

Here is great article about WebGL Skinning: https://webglfundamentals.org/webgl/lessons/webgl-skinning.html

What worked for me when taking vertices/normals from object space to world space is to use the mesh instance’s node world transform matrix - which you can get with this method:

var wmt = entity.model.meshInstances[0].node.getWorldTransform();
1 Like