Pen Settings

HTML

CSS

CSS Base

Vendor Prefixing

Add External Stylesheets/Pens

Any URLs added here will be added as <link>s in order, and before the CSS in the editor. You can use the CSS from another Pen by using its URL and the proper URL extension.

+ add another resource

JavaScript

Babel includes JSX processing.

Add External Scripts/Pens

Any URL's added here will be added as <script>s in order, and run before the JavaScript in the editor. You can use the URL of any other Pen and it will include the JavaScript from that Pen.

+ add another resource

Packages

Add Packages

Search for and use JavaScript packages from npm here. By selecting a package, an import statement will be added to the top of the JavaScript editor for this package.

Behavior

Auto Save

If active, Pens will autosave every 30 seconds after being saved once.

Auto-Updating Preview

If enabled, the preview panel updates automatically as you code. If disabled, use the "Run" button to update.

Format on Save

If enabled, your code will be formatted when you actively save your Pen. Note: your code becomes un-folded during formatting.

Editor Settings

Code Indentation

Want to change your Syntax Highlighting theme, Fonts and more?

Visit your global Editor Settings.

HTML

              
                <div class="container">
  <video class="input_video"></video>
  <canvas class="output_canvas" width="1280px" height="720px"></canvas>
  <div class="loading">
    <div class="spinner"></div>
    <div class="message">
      Loading
    </div>
  </div>
  <a class="abs logo" href="http://www.mediapipe.dev" target="_blank">
    <div style="display: flex;align-items: center;bottom: 0;right: 10px;">
      <img class="logo" src="https://assets.codepen.io/5409376/logo_white.png" alt="" style="
        height: 50px;">
      <span class="title">MediaPipe</span>
    </div>
  </a>
</div>
              
            
!

CSS

              
                @keyframes spin {
  0% { transform: rotate(0deg); }
  100% { transform: rotate(360deg); }
}

.abs {
  position: absolute;
}

a {
  text-decoration: none;
}

body {
  bottom: 0;
  font-family: 'Titillium Web', sans-serif;
  color: white;
  left: 0;
  margin: 0;
  position: absolute;
  right: 0;
  top: 0;
  transform-origin: 0px 0px;
  overflow: hidden;
}

.container {
  position: absolute;
  background-color: #596e73;
  height: 720px;
  width: 1280px;
}

.input_video {
  position:relative;
  top: 0;
  left: 0;
  right: 0;
  bottom: 0;
  &.selfie {
    transform: scale(-1, 1);
  }
}

.output_canvas {
  position:absolute;
  height: 720px;
  width: 1280px;
  left: 0;
  top: 0;
}

.logo {
  bottom: 10px;
  right: 20px;

  .title {
    color: white;
    font-size: 28px;
  }

  .subtitle {
    position: relative;
    color: white;
    font-size: 10px;
    left: -30px;
    top: 20px;
  }
}

.control-panel {
  position: absolute;
  left: 10px;
  top: 10px;
}

.loading {
  display: flex;
  position: absolute;
  top: 0;
  right: 0;
  bottom: 0;
  left: 0;
  align-items: center;
  backface-visibility: hidden;
  justify-content: center;
  opacity: 1;
  transition: opacity 1s;

  .message {
    font-size: x-large;
  }

  .spinner {
    position: absolute;
    width: 120px;
    height: 120px;
    animation: spin 1s linear infinite;
    border: 32px solid #bebebe;
    border-top: 32px solid #3498db;
    border-radius: 50%;
  }
}

.loaded .loading {
  opacity: 0;
}

              
            
!

JS

              
                (function () {
  "use strict";

  // A custom shader that simulates ice surface refraction.
  const iceRefractionShader = {
    uniforms: {
      color: { value: null },
      tDiffuse: { value: null },
      tIceColorMap: { value: null },
      tIceAlphaMap: { value: null },
      tAlphaMap: { value: null },
      tDudv: { value: null },
      textureMatrix: { value: null }
    },

    vertexShader: `
    uniform mat4 textureMatrix;
    varying vec2 vUv;
    varying vec4 vUvRefraction;

    void main() {
      vUv = uv;
      vUvRefraction = textureMatrix * vec4(position, 1.0);
      gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
    }
  `,

    fragmentShader: `
    uniform float time;
    uniform sampler2D tDiffuse;
    uniform sampler2D tIceColorMap;
    uniform sampler2D tIceAlphaMap;
    uniform sampler2D tAlphaMap;
    uniform sampler2D tDudv;

    varying vec2 vUv;
    varying vec4 vUvRefraction;

    void main() {
      float distortionStrength = 0.5;

      vec2 distortedUv = texture2D(tDudv, vUv.xy).rg * distortionStrength;
      distortedUv = vUv.xy + vec2(distortedUv.x, distortedUv.y);
      vec2 distortion = (texture2D(tDudv, distortedUv).rg * 2.0 - 1.0) * distortionStrength;

      vec4 uv = vec4(vUvRefraction);
      uv.xy += distortion;

      vec4 base = texture2DProj(tDiffuse, uv);
      vec4 iceColor = texture2D(tIceColorMap, vUv.xy);
      vec4 iceAlpha = texture2D(tIceAlphaMap, vUv.xy);
      vec4 alpha = texture2D(tAlphaMap, vUv.xy);

      gl_FragColor = vec4(mix(base.rgb, iceColor.rgb, iceAlpha.r), alpha.r);
    }
  `
  };

  class IceDefrostingEffectRenderer {
    constructor(config) {
      // Remember the config.
      this.config = config;

      // Create a scene, a renderer and a camera.
      this.scene = new THREE.Scene();

      this.renderer = new THREE.WebGLRenderer({
        canvas: config.canvasElement,
        antialias: true
      });
      this.renderer.setSize(config.viewportWidth, config.viewportHeight);

      this.camera = new THREE.PerspectiveCamera(
        config.verticalFov,
        config.viewportWidth / config.viewportHeight,
        config.near,
        config.far
      );

      // Create two buffer for frost layer mask accumulation.
      this.currentAccumulationMaskBuffer = new THREE.WebGLRenderTarget(
        config.viewportWidth,
        config.viewportHeight,
        { minFilter: THREE.LinearFilter, magFilter: THREE.NearestFilter }
      );

      this.previousAccumulationMaskBuffer = new THREE.WebGLRenderTarget(
        config.viewportWidth,
        config.viewportHeight,
        { minFilter: THREE.LinearFilter, magFilter: THREE.NearestFilter }
      );

      // Optimization: create commonly used plane geometries.
      this.viewportPlane = new THREE.PlaneBufferGeometry(
        config.viewportWidth,
        config.viewportHeight
      );
      this.unitSizePlane = new THREE.PlaneBufferGeometry(1, 1);

      // Start loading assets need for this effect.
      this.numAssetsToLoad = 6;
      const root = this;
      const onTextureLoadFn = function (_) {
        --root.numAssetsToLoad;
      };

      const texturePathPrefix = "https://assets.codepen.io/5409376";

      this.frostDiffuseTexture = new THREE.TextureLoader().load(
        texturePathPrefix + "/frost_diffuse.jpg",
        onTextureLoadFn
      );

      this.frostDudvTexture = new THREE.TextureLoader().load(
        texturePathPrefix + "/frost_dudv.jpg",
        onTextureLoadFn
      );
      this.frostDudvTexture.wrapS = this.frostDudvTexture.wrapT =
        THREE.RepeatWrapping;

      this.frostTransparencyTexture = new THREE.TextureLoader().load(
        texturePathPrefix + "/frost_transparency.jpg",
        onTextureLoadFn
      );

      this.handHintTexture = new THREE.TextureLoader().load(
        texturePathPrefix + "/hand_hint.png",
        onTextureLoadFn
      );

      this.handMaskTexture = new THREE.TextureLoader().load(
        texturePathPrefix + "/hand_mask.png",
        onTextureLoadFn
      );

      this.onboardingTextTexture = new THREE.TextureLoader().load(
        texturePathPrefix + "/onboarding_text.png",
        onTextureLoadFn
      );

      // Define a safe upper bound for the number of layer-planes. Used to
      // calculate Z coordinates based on the plane order.
      this.numLayersUpperBound = 16;
    }

    isReadyToRender() {
      return this.numAssetsToLoad <= 0;
    }

    render(cameraFrame, firstHandCircle) {
      if (!this.isReadyToRender()) {
        return;
      }

      // Ignore the hand tracking result in the beginning to allow other animated
      // elements to progress.
      this.ignoreHandsAnimationClock =
        this.ignoreHandsAnimationClock || new THREE.Clock();
      if (
        this.ignoreHandsAnimationClock.getElapsedTime() <
        this.config.ignoreHandsIntroLength
      ) {
        firstHandCircle = null;
      }

      // Update (render) the occlusion mask for the frost object.
      this.renderFrostOcclusionMaskPass(firstHandCircle);

      // Render the final effect.
      this.renderFinalPass(cameraFrame, firstHandCircle);

      // Swap the current and the previous accumulation mask buffers.
      [
        this.currentAccumulationMaskBuffer,
        this.previousAccumulationMaskBuffer
      ] = [
        this.previousAccumulationMaskBuffer,
        this.currentAccumulationMaskBuffer
      ];
    }

    renderFrostOcclusionMaskPass(firstHandCircle) {
      // Create planes for the frost layer mask accumulation logic.
      const previousAccumulationMaskPlane = this.placeInPerspectiveFrustum(
        this.createPreviousAccumulationMaskPlane(),
        this.calculateDepthFromOrderIdx(-1)
      );
      this.scene.add(previousAccumulationMaskPlane);

      const deltaAccumulationMaskPlane = this.placeInPerspectiveFrustum(
        this.createDeltaAccumulationMaskPlane(firstHandCircle != null),
        this.calculateDepthFromOrderIdx(-2)
      );
      if (deltaAccumulationMaskPlane) {
        this.scene.add(deltaAccumulationMaskPlane);
      }

      // Create the hand mask object to clear the accumulated mask.
      const handMaskObj = this.placeInPerspectiveFrustum(
        this.createHandMaskObj(firstHandCircle),
        this.calculateDepthFromOrderIdx(-3)
      );
      if (handMaskObj) {
        this.scene.add(handMaskObj);
      }

      this.renderer.setRenderTarget(this.currentAccumulationMaskBuffer);
      this.renderer.render(this.scene, this.camera);
      this.renderer.setRenderTarget(null);
      this.scene.clear();
    }

    renderFinalPass(cameraFrame, firstHandCircle) {
      // Create the camera frame plane.
      const cameraFramePlane = this.placeInPerspectiveFrustum(
        this.createCameraFramePlane(cameraFrame),
        this.calculateDepthFromOrderIdx(-1)
      );
      this.scene.add(cameraFramePlane);

      // Create the ice frost plane.
      const frostPlane = this.placeInPerspectiveFrustum(
        this.createFrostPlane(),
        this.calculateDepthFromOrderIdx(-2)
      );
      this.scene.add(frostPlane);

      // Create the onboarding text plane.
      const onboardingTextPlane = this.placeInPerspectiveFrustum(
        this.createOnboardingTextPlane(firstHandCircle != null),
        this.calculateDepthFromOrderIdx(-4)
      );
      this.scene.add(onboardingTextPlane);

      // Create the hand hint object to show user which hand is tracked.
      const handHintObj = this.placeInPerspectiveFrustum(
        this.createHandHintObj(firstHandCircle),
        this.calculateDepthFromOrderIdx(-3)
      );
      if (handHintObj) {
        this.scene.add(handHintObj);
      }

      this.renderer.render(this.scene, this.camera);
      this.scene.clear();
    }

    createCameraFramePlane(image) {
      return new THREE.Mesh(
        this.viewportPlane,
        new THREE.MeshBasicMaterial({
          map: new THREE.CanvasTexture(image)
        })
      );
    }

    createFrostPlane() {
      const frostPlane = new THREE.Refractor(this.viewportPlane, {
        textureWidth: this.config.viewportWidth,
        textureHeight: this.config.viewportHeight,
        shader: iceRefractionShader
      });

      frostPlane.material.uniforms["tDudv"].value = this.frostDudvTexture;
      frostPlane.material.uniforms[
        "tIceColorMap"
      ].value = this.frostDiffuseTexture;
      frostPlane.material.uniforms[
        "tIceAlphaMap"
      ].value = this.frostTransparencyTexture;
      frostPlane.material.uniforms[
        "tAlphaMap"
      ].value = this.currentAccumulationMaskBuffer.texture;

      return frostPlane;
    }

    createOnboardingTextPlane(hasFirstHand) {
      const config = this.config.onboardingText;
      const opacityAnimationIntroLength = config.opacityAnimationIntroLength;
      const opacityAnimationLoopLength = config.opacityAnimationLoopLength;

      this.onboardingTextAnimationClock =
        this.onboardingTextAnimationClock || new THREE.Clock(false);

      let opacity = null;
      if (hasFirstHand) {
        this.onboardingTextAnimationClock.stop();
        opacity = 0;
      } else {
        if (!this.onboardingTextAnimationClock.running) {
          this.onboardingTextAnimationClock.start();
        }

        const elapsedTime = this.onboardingTextAnimationClock.getElapsedTime();
        if (elapsedTime < opacityAnimationIntroLength) {
          opacity = 0;
        } else {
          const loopTime =
            (elapsedTime - opacityAnimationIntroLength) %
            opacityAnimationLoopLength;
          opacity = (2 * loopTime) / opacityAnimationLoopLength;

          if (opacity > 1) {
            opacity = 2 - opacity;
          }
        }
      }

      return new THREE.Mesh(
        this.viewportPlane,
        new THREE.MeshBasicMaterial({
          map: this.onboardingTextTexture,
          opacity,
          transparent: true
        })
      );
    }

    createPreviousAccumulationMaskPlane() {
      return new THREE.Mesh(
        this.viewportPlane,
        new THREE.MeshBasicMaterial({
          map: this.previousAccumulationMaskBuffer.texture,
          blending: THREE.AdditiveBlending
        })
      );
    }

    createDeltaAccumulationMaskPlane(hasFirstHand) {
      const config = this.config.maskAccumulation;
      const animationIntroLength = config.animationIntroLength;
      const animationGainLength = config.animationGainLength;
      const skipIntroDuringFirstCycle = config.skipIntroDuringFirstCycle;

      this.maskAccumulationAnimationClock =
        this.maskAccumulationAnimationClock || new THREE.Clock(false);
      this.maskAccumulationAnimationNumCycles =
        this.maskAccumulationAnimationNumCycles || 0;

      if (hasFirstHand) {
        this.maskAccumulationAnimationClock.stop();
        return null;
      }

      if (!this.maskAccumulationAnimationClock.running) {
        this.maskAccumulationAnimationClock.start();
        ++this.maskAccumulationAnimationNumCycles;
      }

      const conditionedAnimationIntroLength =
        this.maskAccumulationAnimationNumCycles == 1 &&
        skipIntroDuringFirstCycle
          ? 0
          : animationIntroLength;

      const delta = this.maskAccumulationAnimationClock.getDelta();
      const elapsedTime = this.maskAccumulationAnimationClock.elapsedTime;
      if (elapsedTime < conditionedAnimationIntroLength) {
        return null;
      }

      const maskAccumulationDelta =
        Math.min(delta, elapsedTime - conditionedAnimationIntroLength) /
        animationGainLength;

      return new THREE.Mesh(
        this.viewportPlane,
        new THREE.MeshBasicMaterial({
          color: new THREE.Color(
            maskAccumulationDelta,
            maskAccumulationDelta,
            maskAccumulationDelta
          ),
          blending: THREE.AdditiveBlending,
          transparent: true
        })
      );
    }

    createHandMaskObj(handCircle) {
      if (!handCircle) {
        return null;
      }

      const planeSize = this.config.handMaskRadiusFactor * handCircle.radius;
      const handMaskObj = new THREE.Mesh(
        this.unitSizePlane,
        new THREE.MeshBasicMaterial({
          map: this.handMaskTexture,
          transparent: true
        })
      );

      handMaskObj.scale.x = planeSize;
      handMaskObj.scale.y = planeSize;
      handMaskObj.position.x = handCircle.centerX;
      handMaskObj.position.y = handCircle.centerY;

      return handMaskObj;
    }

    createHandHintObj(handCircle) {
      if (!handCircle) {
        return null;
      }

      const planeSize = this.config.handHintRadiusFactor * handCircle.radius;
      const handHintObj = new THREE.Mesh(
        this.unitSizePlane,
        new THREE.MeshBasicMaterial({
          map: this.handHintTexture,
          transparent: true
        })
      );

      handHintObj.scale.x = planeSize;
      handHintObj.scale.y = planeSize;
      handHintObj.position.x = handCircle.centerX;
      handHintObj.position.y = handCircle.centerY;

      return handHintObj;
    }

    calculateDepthFromOrderIdx(orderIdx) {
      if (
        orderIdx >= this.numLayersUpperBound ||
        orderIdx < -this.numLayersUpperBound
      ) {
        throw new Error(
          `orderIdx is out of range! orderIdx = ${orderIdx}, numLayersUpperBound = ${this.numLayersUpperBound}`
        );
      }

      if (orderIdx < 0) {
        orderIdx += this.numLayersUpperBound;
      }

      const segmentLength =
        (this.config.far - this.config.near) / (this.numLayersUpperBound + 1);

      return this.config.near + segmentLength * (orderIdx + 1);
    }

    placeInPerspectiveFrustum(obj, depth) {
      if (!obj) {
        return obj;
      }

      const heightAtDepth =
        2 * depth * Math.tan((this.config.verticalFov * Math.PI) / 360);
      const scaleFactor = heightAtDepth / this.config.viewportHeight;

      obj.scale.x *= scaleFactor;
      obj.scale.y *= scaleFactor;
      obj.scale.z *= scaleFactor;

      obj.position.x *= scaleFactor;
      obj.position.y *= scaleFactor;
      obj.position.z = obj.position.z * scaleFactor - depth;

      return obj;
    }
  }

  class HandLandmarkProcessor {
    constructor(config) {
      // Remember the config.
      this.config = config;

      // Set all state-related members to default values.
      this.filteredHandCircle = null;
      this.lastHandCircle = null;
      this.lastHandCircleLifetime = 0;
      this.lastHandCirclePresence = 0;
    }

    // Returns a hand circle data for the first tracked hand.
    process(multiHandLandmarks) {
      return this.filterHandCircleTemporally(
        this.extractFirstHandCircle(multiHandLandmarks)
      );
    }

    extractFirstHandCircle(multiHandLandmarks) {
      if (!multiHandLandmarks || multiHandLandmarks.length < 1) {
        return null;
      }

      const handLandmarks = multiHandLandmarks[0];
      const viewportLandmarkX = (i) => {
        return (handLandmarks[i].x - 0.5) * this.config.viewportWidth;
      };
      const viewportLandmarkY = (i) => {
        return (0.5 - handLandmarks[i].y) * this.config.viewportHeight;
      };

      let centerX = 0;
      let centerY = 0;
      for (let i = 0; i < handLandmarks.length; ++i) {
        centerX += (viewportLandmarkX(i) - centerX) / (i + 1);
        centerY += (viewportLandmarkY(i) - centerY) / (i + 1);
      }

      let radius = 0;
      for (let i = 0; i < handLandmarks.length; ++i) {
        const dx = viewportLandmarkX(i) - centerX;
        const dy = viewportLandmarkY(i) - centerY;

        radius = Math.max(radius, Math.sqrt(dx * dx + dy * dy));
      }

      return { centerX, centerY, radius };
    }

    filterHandCircleTemporally(handCircle) {
      const mix = (a, b) => {
        return (
          (1 - this.config.lowPassAlpha) * a + this.config.lowPassAlpha * b
        );
      };
      const mixHandCircle = (a, b) => {
        return {
          centerX: mix(a.centerX, b.centerX),
          centerY: mix(a.centerY, b.centerY),
          radius: mix(a.radius, b.radius)
        };
      };

      if (handCircle) {
        this.lastHandCircle = handCircle;
        this.lastHandCircleLifetime = 0;
        ++this.lastHandCirclePresence;
      } else {
        ++this.lastHandCircleLifetime;
        if (
          this.lastHandCircleLifetime >=
          this.config.handDetectionLifetimeThreshold
        ) {
          this.lastHandCircle = null;
          this.lastHandCircleLifetime = 0;
          this.lastHandCirclePresence = 0;
        }
      }

      if (this.filteredHandCircle && this.lastHandCircle) {
        this.filteredHandCircle = mixHandCircle(
          this.filteredHandCircle,
          this.lastHandCircle
        );
      } else if (this.lastHandCircle) {
        this.filteredHandCircle = this.lastHandCircle;
      } else {
        this.filteredHandCircle = null;
      }

      if (
        this.lastHandCirclePresence >=
        this.config.handDetectionPresenceThreshold
      ) {
        return this.filteredHandCircle;
      } else {
        return null;
      }
    }
  }

  function main() {
    // Our input frames will come from here.
    const videoElement = document.getElementsByClassName("input_video")[0];
    const canvasElement = document.getElementsByClassName("output_canvas")[0];

    // Optimization: Turn off animated spinner after its hiding animation is done.
    const spinner = document.querySelector(".loading");
    spinner.ontransitionend = () => {
      spinner.style.display = "none";
    };

    // Define demo config.
    const iceDefrostingEffectConfig = {
      canvasElement: canvasElement,
      viewportWidth: 1280,
      viewportHeight: 720,
      verticalFov: 45,
      near: 1,
      far: 100,
      handMaskRadiusFactor: 2.5,
      handHintRadiusFactor: 2,
      // Animation lengths are in seconds.
      ignoreHandsIntroLength: 2,
      maskAccumulation: {
        animationIntroLength: 0.75,
        animationGainLength: 3,
        skipIntroDuringFirstCycle: true
      },
      onboardingText: {
        opacityAnimationIntroLength: 1,
        opacityAnimationLoopLength: 2
      }
    };

    const handLandmarkProcessorConfig = {
      viewportWidth: iceDefrostingEffectConfig.viewportWidth,
      viewportHeight: iceDefrostingEffectConfig.viewportHeight,
      lowPassAlpha: 0.8,
      handDetectionLifetimeThreshold: 3,
      handDetectionPresenceThreshold: 3
    };

    const handTrackerConfig = {
      selfieMode: true,
      maxHands: 1,
      minDetectionConfidence: 0.7,
      minTrackingConfidence: 0.9
    };

    // Update the camera video element to reflect the selfie mode option.
    videoElement.classList.toggle("selfie", handTrackerConfig.selfieMode);

    // Create an ice defrosting demo renderer.
    const iceDefrostingEffectRenderer = new IceDefrostingEffectRenderer(
      iceDefrostingEffectConfig
    );

    // Create a hand landmark processor.
    const handLandmarkProcessor = new HandLandmarkProcessor(
      handLandmarkProcessorConfig
    );

    // Create a hand tracker.
    const handTracker = new Hands({
      locateFile: (x) => {
        return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${x}`;
      }
    });
    handTracker.setOptions(handTrackerConfig);
    handTracker.onResults((results) => {
      if (!iceDefrostingEffectRenderer.isReadyToRender()) {
        return;
      }

      // Hide the spinner.
      document.body.classList.add("loaded");

      // Process landmarks & render the effect.
      iceDefrostingEffectRenderer.render(
        results.image,
        handLandmarkProcessor.process(results.multiHandLandmarks)
      );
    });

    // Instantiate a camera. We'll feed each frame we receive into the solution.
    const cameraMp = new Camera(videoElement, {
      onFrame: async () => {
        await handTracker.send({ image: videoElement });
      },
      width: iceDefrostingEffectConfig.viewportWidth,
      height: iceDefrostingEffectConfig.viewportHeight
    });
    cameraMp.start();
  }

  main();
})();

              
            
!
999px

Console