[SOLVED] Mesh outline from editor

Edit: I threw away this approach and just used cloned mesh with frontface culling and pure black shader with vertex offset by normal.

Hi! I am trying to do a mesh outline on user hover. I tried to take the code from Playcanvas’ editor and use it in my script with a bit o editing, but I was unable to make it work (like literally, nothing is happening - no error either). Would you be willing to take a look and tell me what could be wrong? Here is the code I am using for now:

(The entity gets picked fine, the problem is in the “rendering” I guess, I am kind of lost in the layer/comp system)

Outline.prototype.initialize = function() {
    var camera = this.app.root.findByName("MainCamera").camera.camera;
    var self = this;
    var app = this.app;
    var renderer = this.app.renderer;
    this.device = this.app.renderer.device;
    var scene = this.app.scene;

    this.selection = { };
    var colors = { };
    this.render = 0;
    var cleared = false;
    this.visible = true;

    this.targets = [ ];
    this.textures = [ ];

    this.SHADER_OUTLINE = 24;
    
    var outlined = false;
    
    this.app.mouse.on(pc.EVENT_MOUSEMOVE, function(e) {
        var cameraComp = self.app.root.findByName("MainCamera").camera;
        var camera = cameraComp.camera;
        var from = cameraComp.screenToWorld(e.x, e.y, camera.nearClip);
        var to = cameraComp.screenToWorld(e.x, e.y, camera.farClip);

        var result = self.app.systems.rigidbody.raycastFirst(from, to);

        if (result) {            
            var pickedEntity = result.entity;
            if(self.entity == pickedEntity && !outlined) {
                console.log(config.self.id);
                console.log("Hovering over this entity!");
                self.selection[config.self.id] = [ ];
                self.selection[config.self.id].push(self.entity);
                self.render++;
                outlined = true;                
            }
        } else if (!result && outlined) {
            outlined = false;
        }
    }, this);

    // ### OVERLAY QUAD MATERIAL ###
    var chunks = pc.shaderChunks;
    var shaderFinal = chunks.createShaderFromCode(self.device, chunks.fullscreenQuadVS, chunks.outputTex2DPS, "outputTex2D");

    // ### OUTLINE EXTEND SHADER H ###
    var shaderBlurHPS = ' \
        precision ' + self.device.precision + ' float;\n \
        varying vec2 vUv0;\n \
        uniform float uOffset;\n \
        uniform sampler2D source;\n \
        void main(void)\n \
        {\n \
            float diff = 0.0;\n \
            vec4 pixel;\n \
            vec4 texel = texture2D(source, vUv0);\n \
            vec4 firstTexel = texel;\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(uOffset * -2.0, 0.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(uOffset * -1.0, 0.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(uOffset * +1.0, 0.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(uOffset * +2.0, 0.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            gl_FragColor = vec4(texel.rgb, min(diff, 1.0));\n \
        }\n';
    var shaderBlurH = chunks.createShaderFromCode(self.device, chunks.fullscreenQuadVS, shaderBlurHPS, "editorOutlineH");

    // ### OUTLINE EXTEND SHADER V ###
    var shaderBlurVPS = ' \
        precision ' + self.device.precision + ' float;\n \
        varying vec2 vUv0;\n \
        uniform float uOffset;\n \
        uniform sampler2D source;\n \
        void main(void)\n \
        {\n \
            vec4 pixel;\n \
            vec4 texel = texture2D(source, vUv0);\n \
            vec4 firstTexel = texel;\n \
            float diff = texel.a;\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(0.0, uOffset * -2.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(0.0, uOffset * -1.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(0.0, uOffset * +1.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            pixel = texture2D(source, vUv0 + vec2(0.0, uOffset * +2.0));\n \
            texel = max(texel, pixel);\n \
            diff = max(diff, length(firstTexel.rgb - pixel.rgb));\n \
            \n \
            gl_FragColor = vec4(texel.rgb, min(diff, 1.0));\n \
        }\n';
    var shaderBlurV = chunks.createShaderFromCode(self.device, chunks.fullscreenQuadVS, shaderBlurVPS, "editorOutlineV");


    self.outlineLayer = new pc.Layer({
        name: "Outline",
        opaqueSortMode: pc.SORTMODE_NONE,
        passThrough: true,
        overrideClear: true,
        clearColorBuffer: false,
        clearDepthBuffer: false,
        clearColor: new pc.Color(0,0,0,0),
        shaderPass: self.SHADER_OUTLINE,

        onPostRender: function() {
            // extend pass X
            var uOffset = self.device.scope.resolve('uOffset');
            var uColorBuffer = self.device.scope.resolve('source');
            uOffset.setValue(1.0 / self.device.width / 2.0);
            uColorBuffer.setValue(self.textures[0]);
            pc.drawQuadWithShader(self.device, self.targets[1], shaderBlurH);

            // extend pass Y
            uOffset.setValue(1.0 / self.device.height / 2.0);
            uColorBuffer.setValue(self.textures[1]);
            pc.drawQuadWithShader(self.device, self.targets[0], shaderBlurV);
        }
    });
    self.outlineComp = new pc.LayerComposition();
    self.outlineComp.pushOpaque(self.outlineLayer);

    self.onUpdateShaderOutline = function(options) {
        if (options.pass !== self.SHADER_OUTLINE) return options;
        var outlineOptions = {
            opacityMap:                 options.opacityMap,
            opacityMapUv:               options.opacityMapUv,
            opacityMapChannel:          options.opacityMapChannel,
            opacityMapTransform:        options.opacityMapTransform,
            opacityVertexColor:         options.opacityVertexColor,
            opacityVertexColorChannel:  options.opacityVertexColorChannel,
            vertexColors:               options.vertexColors,
            alphaTest:                  options.alphaTest,
            skin:                       options.skin
        }
        return outlineOptions;
    };    
   
};

// ### RENDER EVENT ###
Outline.prototype.update = function() {
    var camera = this.app.root.findByName("MainCamera").camera.camera;
    var cameraComp = this.app.root.findByName("MainCamera").camera;

    // ### INIT/RESIZE RENDERTARGETS ###
    if (this.targets[0] && (this.targets[0].width !== this.device.width || this.targets[1].height !== this.device.height)) {
        for(var i = 0; i < 2; i++) {
            this.targets[i].destroy();
            this.textures[i].destroy();
        }
        this.targets = [ ];
        this.textures = [ ];
    }
    if (! this.targets[0]) {
        for(var i = 0; i < 2; i++) {
            this.textures[i] = new pc.Texture(this.device, {
                format: pc.PIXELFORMAT_R8_G8_B8_A8,
                width: this.device.width,
                height: this.device.height
            });
            this.textures[i].minFilter = pc.FILTER_NEAREST;
            this.textures[i].magFilter = pc.FILTER_NEAREST;
            this.textures[i].addressU = pc.ADDRESS_CLAMP_TO_EDGE;
            this.textures[i].addressV = pc.ADDRESS_CLAMP_TO_EDGE;

            this.targets[i] = new pc.RenderTarget(this.device, this.textures[i]);
        }
    }

    if (this.render) {
        // ### RENDER COLORED MESHINSTANCES TO RT0 ###

        this.outlineLayer.renderTarget = this.targets[0];
        this.outlineLayer.clearMeshInstances();
        if (this.outlineLayer.cameras[0] !== cameraComp) {
            this.outlineLayer.clearCameras();
            this.outlineLayer.addCamera(cameraComp);
            console.log(this.outlineLayer.cameras);
        }
        var meshInstances = this.outlineLayer.opaqueMeshInstances;

        if (this.visible) {
            var color = new pc.Color(255, 0, 0);

            for(var i = 0; i < this.selection[config.self.id].length; i++) {
                if (! this.selection[config.self.id][i])
                    continue;

                var model = this.selection[config.self.id][i].model;
                if (! model || ! model.model)
                    continue;

                var meshes = model.meshInstances;
                for(var m = 0; m < meshes.length; m++) {
                    //var opChan = 'r';
                    var instance = meshes[m];

                    //if (! instance.command && instance.drawToDepth && instance.material && instance.layer === pc.LAYER_WORLD) {
                    if (!instance.command && instance.material) {
                        instance.onUpdateShader = this.onUpdateShaderOutline;
                        instance.setParameter("material_emissive", [color.r, color.g, color.b], 1<<this.SHADER_OUTLINE);
                        meshInstances.push(instance);
                    }
                }
            }            
        }

        this.app.renderer.renderComposition(this.outlineComp);
    }
};